Commit d76c41e6 authored by ismael.rodriguez's avatar ismael.rodriguez

Cleaning nonparametric tests

parent cca084b1
......@@ -7,7 +7,7 @@ RUN apt-get update && apt-get install -y git
RUN git clone https://gitlab.citius.usc.es/ismael.rodriguez/stac.git
RUN mkdir /var/www/stac
RUN cp stac/api stac/web stac/stac /var/www/stac/
RUN cp -R stac/api stac/web stac/stac /var/www/stac/
RUN chmod -R 755 /var/www/stac
ADD stac.conf /etc/apache2/sites-enabled/000-default.conf
......
......@@ -10,117 +10,69 @@ import scipy.stats as st
import numpy as np
from stac import tests_no_parametricos as tnp
from stac import *
from utils import LimitedSizeDict, leer_datos, generar_md5
from utils import LimitedSizeDict, leer_datos, generar_md5, clean_missing_values
import json
lista_ficheros = LimitedSizeDict(size_limit=5)
#Servicio para la subida de ficheros.
@route('/file', method='POST')
def post_file():
response.headers['Access-Control-Allow-Origin'] = '*'
response.content_type = "application/json"
subida = request.files.get('file')
clave_hash = generar_md5(subida.file)
for clave in lista_ficheros.keys():
if clave == clave_hash:
return {"hash" : clave_hash}
try:
datos = leer_datos(subida.file)
except Exception, fallo:
return {"fallo" : str(fallo)}
lista_ficheros[clave_hash] = datos
#Servicio para el test de Wilcoxon.
#@route('/wilcoxon/<id_fichero>', method="GET")
#@route('/wilcoxon/<id_fichero>/<alpha:float>', method="GET")
#def wilcoxon(id_fichero, alpha=0.05):
# response.headers['Access-Control-Allow-Origin'] = '*'
# response.content_type = "application/json"
# try:
# datos = lista_ficheros[id_fichero]
# except Exception:
# return {"error" : "There is no file with that key."}
# try:
# statistic, p_value = st.wilcoxon([v[0] for v in datos["matriz_datos"]], [v[1] for v in datos["matriz_datos"]])
# result = np.asscalar(p_value<alpha)
# except Exception, fallo:
# return {"error" : str(fallo)}
# return json.dumps({"result" : result, "statistic" : statistic, "p_value" : p_value})
return {"hash" : clave_hash}
#Servicio para la consulta de ficheros.
@route('/file/<id_fichero>', method='GET')
def get_file(id_fichero):
@route('/wilcoxon', method="POST")
@route('/wilcoxon/<alpha:float>', method="POST")
def wilcoxon(alpha=0.05):
response.headers['Access-Control-Allow-Origin'] = '*'
response.content_type = "application/json"
#Consulta del contenido de un fichero en concreto.
values = clean_missing_values(request.json['values'])
try:
datos = lista_ficheros[id_fichero]
except Exception:
return {"fallo" : "There is no file with that key."}
return datos
#Servicio para el test de Wilcoxon.
@route('/wilcoxon/<id_fichero>', method="GET")
@route('/wilcoxon/<id_fichero>/<alpha:float>', method="GET")
def wilcoxon(id_fichero, alpha=0.05):
response.headers['Access-Control-Allow-Origin'] = '*'
response.content_type = "application/json"
try:
datos = lista_ficheros[id_fichero]
except Exception:
return {"error" : "There is no file with that key."}
try:
statistic, p_value = st.wilcoxon([v[0] for v in datos["matriz_datos"]], [v[1] for v in datos["matriz_datos"]])
result = np.asscalar(p_value<alpha)
except Exception, fallo:
return {"error" : str(fallo)}
return json.dumps({"result" : result, "statistic" : statistic, "p_value" : p_value})
@route('/mannwhitneyu/<id_fichero>', method="GET")
@route('/mannwhitneyu/<id_fichero>/<alpha:float>', method="GET")
def mannwhitneyu(id_fichero, alpha=0.05):
response.headers['Access-Control-Allow-Origin'] = '*'
response.content_type = "application/json"
try:
datos = lista_ficheros[id_fichero]
except Exception:
return {"error" : "There is no file with that key."}
try:
statistic, p_value = st.mannwhitneyu([v[0] for v in datos["matriz_datos"]], [v[1] for v in datos["matriz_datos"]], "false")
result = np.asscalar(p_value*2<alpha)
except Exception, fallo:
statistic, p_value = st.wilcoxon(values.values()[0], values.values()[1])
result = np.asscalar(p_value<alpha)
except Exception, fallo:
return {"error" : str(fallo)}
return json.dumps({"result" : result, "statistic" : statistic, "p_value" : p_value*2})
#Servicio para el test de Friedman.
@route('/friedman/<id_fichero>/<test_comparacion>', method="GET")
@route('/friedman/<id_fichero>/<alpha:float>/<test_comparacion>', method="GET")
@route('/friedman/<id_fichero>/<tipo:int>/<test_comparacion>', method="GET")
@route('/friedman/<id_fichero>/<alpha:float>/<tipo:int>/<test_comparacion>', method="GET")
@route('/friedman/<id_fichero>', method="GET")
@route('/friedman/<id_fichero>/<alpha:float>', method="GET")
@route('/friedman/<id_fichero>/<tipo:int>', method="GET")
@route('/friedman/<id_fichero>/<alpha:float>/<tipo:int>', method="GET")
def friedman(id_fichero, alpha=0.05, tipo=0, test_comparacion="bonferroni_dunn_test"):
return json.dumps({"result" : result, "statistic" : statistic, "p_value" : p_value})
@route('/mannwhitneyu/', method="POST")
@route('/mannwhitneyu/<alpha:float>', method="POST")
def mannwhitneyu(alpha=0.05):
response.headers['Access-Control-Allow-Origin'] = '*'
response.content_type = "application/json"
values = clean_missing_values(request.json['values'], delete_row=False)
try:
datos = lista_ficheros[id_fichero]
except Exception:
return {"fallo" : "There is no file with that key."}
resultado = test_ranking(friedman_test, getattr(tnp, test_comparacion), datos["nombres_algoritmos"], datos["matriz_datos"], len(datos["matriz_datos"]), alpha, tipo)
return json.dumps(resultado)
statistic, p_value = st.mannwhitneyu(values.values()[0], values.values()[1], use_continuity="false")
result = np.asscalar(p_value*2<alpha)
except Exception, fallo:
return {"error" : str(fallo)}
return json.dumps({"result" : result, "statistic" : statistic, "p_value" : p_value*2})
#Servicio para el test de Iman-Davenport.
@route('/iman-davenport/<id_fichero>/<test_comparacion>', method="GET")
@route('/iman-davenport/<id_fichero>/<alpha:float>/<test_comparacion>', method="GET")
@route('/iman-davenport/<id_fichero>/<tipo:int>/<test_comparacion>', method="GET")
@route('/iman-davenport/<id_fichero>/<alpha:float>/<tipo:int>/<test_comparacion>', method="GET")
@route('/iman-davenport/<id_fichero>', method="GET")
@route('/iman-davenport/<id_fichero>/<alpha:float>', method="GET")
@route('/iman-davenport/<id_fichero>/<tipo:int>', method="GET")
@route('/iman-davenport/<id_fichero>/<alpha:float>/<tipo:int>', method="GET")
def iman_davenport(id_fichero, alpha=0.05, tipo=0, test_comparacion="bonferroni_dunn_test"):
@route('/friedman/<post_hoc>', method="POST")
@route('/friedman/<alpha:float>/<post_hoc>', method="POST")
@route('/friedman/<objective:int>/<post_hoc>', method="POST")
@route('/friedman/<alpha:float>/<objective:int>/<post_hoc>', method="POST")
def friedman(id_fichero, alpha=0.05, obj=0, post_hoc="bonferroni_dunn_test"):
response.headers['Access-Control-Allow-Origin'] = '*'
response.content_type = "application/json"
try:
datos = lista_ficheros[id_fichero]
except Exception:
return {"fallo" : "There is no file with that key."}
resultado = test_ranking(iman_davenport_test, getattr(tnp, test_comparacion), datos["nombres_algoritmos"], datos["matriz_datos"], len(datos["matriz_datos"]), alpha, tipo)
resultado = test_ranking(friedman_test, getattr(tnp, post_hoc), clean_missing_values(request.json['values']), alpha, objective)
return json.dumps(resultado)
......
......@@ -100,4 +100,22 @@ def generar_md5(archivo):
bufer = archivo.read(tam_bloque)
archivo.seek(0, 0);
return md5.hexdigest()
def clean_missing_values(values, delete_row=True):
n = len(values.values()[0])
names = values.keys()
p = len(names)
if delete_row:
for i in range(n):
for j in range(p):
try:
float(values[names[j]][i])
except:
for k in names:
values[k][i] = float('nan')
break
for k in names:
values[k] = [v for v in values[k] if str(v) != 'nan']
return values
This diff is collapsed.
......@@ -2,43 +2,59 @@ $(document).ready(function() {
$(document).on('click', '#upload_file', function() {
var formData = new FormData($('#formfile')[0]);
var fichero = $("#file").val();
$('#info_file').html("Loading...");
if(!fichero)
$('#info_file').html("<br><p style=\"color:red\";><strong>Select a file</strong></p>");
else {
$.ajax({
type: "POST",
url: APP_CONFIG.api_url + "/file",
dataType: "json",
data: formData,
cache: false,
contentType: false,
processData: false,
success : function(data) {
resultado = "";
if(!data.fallo){
resultado = "<p style=\"color:green\";><strong>Fichero subido con éxito</strong></p>";
sessionStorage.setItem("data", data.hash);
console.log(data);
sessionStorage.setItem("homocedasticity", false);
sessionStorage.setItem("normality", false);
$("#show_file").show();
$('#info_file').html("Done!");
var csvfile = $("#file")[0].files[0];
Papa.parse(csvfile, {
header: true,
dynamicTyping: true,
complete: function(results) {
var data = {};
data["dataset"] = [];
if (results.meta.fields[0] == "dataset") {
results.meta.fields = results.meta.fields.splice(1)
results.data.forEach(function(row) {
data["dataset"].push(row["dataset"]);
});
}
data["names"] = results.meta.fields;
var values = {};
results.meta.fields.forEach(function(field) {
values[field] = [];
});
try {
results.data.forEach(function(row) {
results.meta.fields.forEach(function(field) {
if (row[field]) {
if (row[field] == "-" ||
row[field] == "?" ||
row[field] == "")
{
values[field].push(NaN);
} else if (!isNaN(row[field])) {
values[field].push(row[field]);
} else {
throw "Unexpected character " + row[field];
}
}
});
});
data["values"] = values;
sessionStorage["data"] = JSON.stringify(data);
$("#show_file").show();
$('#info_file').html("Done!");
window.location = APP_CONFIG.app_url + "/file.html";
} else{
resultado = resultado + "<p style=\"color:red\";><strong>" + data.fallo + "</strong></p>";
$('#info_file').html("<br>"+resultado);
$('#formfile').trigger('reset');
}
},
error : function(e) {
console.log(e);
}
});
}
window.location = APP_CONFIG.app_url + "/file.html";
} catch (err) {
var resultado = "<p style=\"color:red\";><strong>" + err + "</strong></p>";
$('#info_file').html("<br>"+resultado);
$('#formfile').trigger('reset');
}
}
});
});
if ($(document).find("#file_table").length > 0) {
......@@ -47,38 +63,35 @@ $(document).ready(function() {
});
function show_file() {
var hash = sessionStorage.getItem("data");
$.ajax({
type: "GET",
url: APP_CONFIG.api_url + "/file/"+hash,
success : function(data) {
if(data.fallo) {
$("#danger").html("There is no file uploaded.").show();
} else {
var salida = "<thead><tr><th>" + data.palabra + "</th>";
var data = JSON.parse(sessionStorage["data"]);
$.each(data.nombres_algoritmos, function(index, value) {
salida = salida + "<th>" + value + "</th>";
});
salida = salida + "</tr></thead><tbody>";
$.each(data.nombres_conj_datos, function(index, value) {
salida = salida + "<tr><td>" + value + "</td>";
$.each(data.matriz_datos[index], function(index, value) {
salida = salida + "<td>" + value + "</td>";
});
salida = salida + "</tr>";
});
salida = salida + "</tbody>";
$('#file_table').html(salida);
}
},
error : function(e) {
console.log('Error: ' + e);
}
});
if (!data) {
$("#danger").html("There is no file uploaded.").show();
} else {
var salida = "<thead><tr>";
if (data.dataset.length != 0) {
salida = salida + "<th>Dataset</th>";
}
$.each(data.names, function(index, value) {
salida = salida + "<th>" + value + "</th>";
});
salida = salida + "</tr></thead><tbody>";
for (var i = 0; i < data.values[data.names[0]].length; i++) {
salida = salida + "<tr>";
if (data.dataset.length != 0) {
salida = salida + "<td>" + data.dataset[i] + "</td>";
}
$.each(data.names, function(j, name) {
salida = salida + "<td>" + data.values[name][i] + "</td>";
});
salida = salida + "</tr>";
}
salida = salida + "</tbody>";
$('#file_table').html(salida);
}
}
......@@ -38,6 +38,7 @@ loadStyle("css/dashboard.css");
loadScript("js/jquery.min.js", function() {
loadScript("js/config.js", function() {
loadScript("js/bootstrap.min.js");
loadScript("js/papaparse.js");
loadScript("js/layout.js");
loadScript("js/behavior.js");
loadScript("js/file_manager.js");
......@@ -45,4 +46,3 @@ loadScript("js/jquery.min.js", function() {
loadScript("js/export.js");
})
});
This diff is collapsed.
......@@ -8,10 +8,9 @@ $(document).ready(function(){
var test = $('input[name=test]:checked').val();
var alpha = $('#alpha').val();
var url = APP_CONFIG.api_url+"/"+test+"/"+sessionStorage.getItem("data")+"/"+alpha;
var url = APP_CONFIG.api_url+"/"+test+"/"+alpha;
var post_hoc = $('input[name=post_hoc]:checked').val();
if (post_hoc) var url = APP_CONFIG.api_url+"/"+test+"/"+sessionStorage.getItem("data")+"/"+alpha+"/"+post_hoc;
console.log(url);
switch (type) {
case "normality":
......@@ -84,8 +83,8 @@ $(document).ready(function(){
$("#danger").hide();
$("#warning").hide();
if (data.fallo) {
$("#danger").html(data.fallo).show();
if (data.error) {
$("#danger").html(data.error).show();
} else {
var salida = ttest_table(data, test, alpha);
......@@ -100,17 +99,17 @@ $(document).ready(function(){
break;
case "wilcoxon":
$.ajax({
type: "GET", url: url, dataType: "json",
type: "POST", url: url, dataType: "json",
contentType: "application/json",
data: sessionStorage.data,
success : function(data) {
$("#danger").hide();
$("#warning").hide();
if (data.error) {
$("#danger").html(data.fallo).show();
$("#danger").html(data.error).show();
} else {
var salida = wilcoxon_table(data, test, alpha);
$("#result").html(salida).show();
$("#result").html(wilcoxon_table(data, test, alpha)).show();
}
},
......@@ -121,13 +120,15 @@ $(document).ready(function(){
break;
case "mannwhitneyu":
$.ajax({
type: "GET", url: url, dataType: "json",
type: "POST", url: url, dataType: "json",
contentType: "application/json",
data: sessionStorage.data,
success : function(data) {
$("#danger").hide();
$("#warning").hide();
if (data.error) {
$("#danger").html(data.fallo).show();
$("#danger").html(data.error).show();
} else {
var salida = wilcoxon_table(data, test, alpha);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment