From fc7cb08469c62ead7f0008487f46982f469ad287 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 8 Sep 2022 14:15:12 -0700 Subject: [PATCH 1/2] Data uploader --- .../app/migrations/0003_uploadeddata.py | 22 ++++++ pgml-dashboard/app/models.py | 36 +++++++++- pgml-dashboard/app/static/css/base.css | 33 +++++++++ pgml-dashboard/app/templates/base.html | 8 +++ .../app/templates/projects/sample.html | 52 +++++++------- .../app/templates/uploader/index.html | 52 ++++++++++++++ .../app/templates/uploader/uploaded.html | 25 +++++++ pgml-dashboard/app/urls.py | 4 +- pgml-dashboard/app/views/uploader.py | 72 +++++++++++++++++++ 9 files changed, 277 insertions(+), 27 deletions(-) create mode 100644 pgml-dashboard/app/migrations/0003_uploadeddata.py create mode 100644 pgml-dashboard/app/templates/uploader/index.html create mode 100644 pgml-dashboard/app/templates/uploader/uploaded.html create mode 100644 pgml-dashboard/app/views/uploader.py diff --git a/pgml-dashboard/app/migrations/0003_uploadeddata.py b/pgml-dashboard/app/migrations/0003_uploadeddata.py new file mode 100644 index 000000000..73a945ddd --- /dev/null +++ b/pgml-dashboard/app/migrations/0003_uploadeddata.py @@ -0,0 +1,22 @@ +# Generated by Django 4.0.7 on 2022-09-08 20:07 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("app", "0002_notebook_notebookcell"), + ] + + operations = [ + migrations.CreateModel( + name="UploadedData", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("file_type", models.IntegerField(choices=[(1, "CSV"), (2, "JSON")], default=1)), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ], + ), + ] diff --git a/pgml-dashboard/app/models.py b/pgml-dashboard/app/models.py index 697955c7a..01d87ac78 100644 --- a/pgml-dashboard/app/models.py +++ b/pgml-dashboard/app/models.py @@ -1,4 +1,4 @@ -from django.db import models, connection +from django.db import models, connection, transaction from django.template.loader import render_to_string from django.utils.safestring import mark_safe from django.db.utils import ProgrammingError @@ -6,6 +6,8 @@ from django.utils.html import strip_tags import markdown +import codecs +import csv class Project(models.Model): @@ -298,3 +300,35 @@ def code(self): def __str__(self): return f"{self.notebook} - {self.pk}" + + +class UploadedData(models.Model): + """Data uploaded by the user through the dashboard.""" + + file_type = models.IntegerField( + choices=( + ( + 1, + "CSV", + ), + (2, "JSON"), + ), + default=1, + ) + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + + def create_table(self, file): + if file.content_type == "text/csv": + reader = csv.reader(codecs.iterdecode(file, "utf-8")) + headers = next(reader) + columns = ", ".join(map(lambda x: f"{x.replace(' ', '_').lower()} FLOAT4", headers)) + + with transaction.atomic(): + sql = f"CREATE TABLE data_{self.pk} (" + columns + ")" + + with connection.cursor() as cursor: + cursor.execute(sql) + + file.seek(0) + cursor.copy_expert(f"COPY data_{self.pk} FROM STDIN CSV HEADER", file) diff --git a/pgml-dashboard/app/static/css/base.css b/pgml-dashboard/app/static/css/base.css index 6d92608ef..ef09a57ce 100644 --- a/pgml-dashboard/app/static/css/base.css +++ b/pgml-dashboard/app/static/css/base.css @@ -652,3 +652,36 @@ main turbo-frame:first-of-type .notebook-cell { .CodeMirror { font-size: 1rem; } + +/* + * Uploader + */ +body.uploader section p, body.uploader section li { + margin: 0.5rem 0; +} + +body.uploader section ol, body.uploader section ul { + margin: 1rem 0; +} + +body.uploader section .markdown-body{ + margin: 1rem 0; +} + +body.uploader ul { + list-style-type: disc; + list-style-position: inside; +} + +body.uploader ol { + list-style-type: decimal; + list-style-position: inside; +} + +body.uploader section li { + margin-left: 1rem; +} + +body.uploader strong { + font-weight: bold; +} diff --git a/pgml-dashboard/app/templates/base.html b/pgml-dashboard/app/templates/base.html index 8f06b33d6..79eac7ae6 100644 --- a/pgml-dashboard/app/templates/base.html +++ b/pgml-dashboard/app/templates/base.html @@ -32,6 +32,13 @@ + + + + + + +