Data uploader (#309)

levkk · web-flow · commit 0645c74bd43e · 2022-09-08T14:28:31.000-07:00
diff --git a/pgml-dashboard/app/migrations/0003_uploadeddata.py b/pgml-dashboard/app/migrations/0003_uploadeddata.py
@@ -0,0 +1,22 @@
+# Generated by Django 4.0.7 on 2022-09-08 20:07
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("app", "0002_notebook_notebookcell"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="UploadedData",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("file_type", models.IntegerField(choices=[(1, "CSV"), (2, "JSON")], default=1)),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("updated_at", models.DateTimeField(auto_now=True)),
+            ],
+        ),
+    ]
diff --git a/pgml-dashboard/app/models.py b/pgml-dashboard/app/models.py
@@ -1,11 +1,13 @@
-from django.db import models, connection
+from django.db import models, connection, transaction
 from django.template.loader import render_to_string
 from django.utils.safestring import mark_safe
 from django.db.utils import ProgrammingError
 from django.utils import timezone
 from django.utils.html import strip_tags
 
 import markdown
+import codecs
+import csv
 
 
 class Project(models.Model):
@@ -298,3 +300,35 @@ def code(self):
 
     def __str__(self):
         return f"{self.notebook} - {self.pk}"
+
+
+class UploadedData(models.Model):
+    """Data uploaded by the user through the dashboard."""
+
+    file_type = models.IntegerField(
+        choices=(
+            (
+                1,
+                "CSV",
+            ),
+            (2, "JSON"),
+        ),
+        default=1,
+    )
+    created_at = models.DateTimeField(auto_now_add=True)
+    updated_at = models.DateTimeField(auto_now=True)
+
+    def create_table(self, file):
+        if file.content_type == "text/csv":
+            reader = csv.reader(codecs.iterdecode(file, "utf-8"))
+            headers = next(reader)
+            columns = ", ".join(map(lambda x: f"{x.replace(' ', '_').lower()} FLOAT4", headers))
+
+            with transaction.atomic():
+                sql = f"CREATE TABLE data_{self.pk} (" + columns + ")"
+
+                with connection.cursor() as cursor:
+                    cursor.execute(sql)
+
+                    file.seek(0)
+                    cursor.copy_expert(f"COPY data_{self.pk} FROM STDIN CSV HEADER", file)
diff --git a/pgml-dashboard/app/static/css/base.css b/pgml-dashboard/app/static/css/base.css
@@ -652,3 +652,36 @@ main turbo-frame:first-of-type .notebook-cell {
 .CodeMirror {
   font-size: 1rem;
 }
+
+/*
+ * Uploader
+ */
+body.uploader section p, body.uploader section li {
+  margin: 0.5rem 0;
+}
+
+body.uploader section ol, body.uploader section ul {
+  margin: 1rem 0;
+}
+
+body.uploader section .markdown-body{
+  margin: 1rem 0;
+}
+
+body.uploader ul { 
+   list-style-type: disc; 
+   list-style-position: inside; 
+}
+
+body.uploader ol { 
+   list-style-type: decimal; 
+   list-style-position: inside; 
+}
+
+body.uploader section li {
+  margin-left: 1rem;
+}
+
+body.uploader strong {
+  font-weight: bold;
+}
diff --git a/pgml-dashboard/app/templates/base.html b/pgml-dashboard/app/templates/base.html
@@ -32,6 +32,13 @@
         <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/codemirror/6.65.7/codemirror.min.css" integrity="sha512-uf06llspW44/LZpHzHT6qBOIVODjWtv4MxCricRxkzvopAlSWnTf6hpZTFxuuZcuNE9CBQhqE0Seu1CoRk84nQ==" crossorigin="anonymous" referrerpolicy="no-referrer" />
         <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/codemirror/6.65.7/addon/hint/show-hint.min.css" integrity="sha512-OmcLQEy8iGiD7PSm85s06dnR7G7C9C0VqahIPAj/KHk5RpOCmnC6R2ob1oK4/uwYhWa9BF1GC6tzxsC8TIx7Jg==" crossorigin="anonymous" referrerpolicy="no-referrer" />
 
+        <!-- Papa Parse <3 -->
+        <!-- <script defer async src="https://cdn.jsdelivr.net/npm/papaparse@5.3.2/papaparse.min.js"></script> -->
+
+        <!-- CSV preview -->
+        <!-- <link rel="stylesheet" href="https://cdn.jsdelivr.net/gh/kktsvetkov/heiho@latest/heiho.css" /> -->
+        <!-- <script async defer src="https://cdn.jsdelivr.net/gh/kktsvetkov/heiho@latest/heiho.js"></script> -->
+
         <script defer src="https://unpkg.com/es-module-shims@1.2.0/dist/es-module-shims.js"></script>
         <script type="importmap-shim">
             {
@@ -92,6 +99,7 @@
                     <li{% if topic == "deployments" %} class="selected"{% endif %}><a href="{% url 'deployments' %}"><span class="material-symbols-outlined">inventory</span>Deployments</a></li>
                     <li{% if topic == "snapshots" %} class="selected"{% endif %}><a href="{% url 'snapshots' %}"><span class="material-symbols-outlined">storage</span>Snapshots</a></li>
                     <li{% if topic == "console" %} class="selected"{% endif %}><a href="{% url 'console' %}"><span class="material-symbols-outlined">terminal</span>Console</a></li>
+                    <li{% if topic == "uploader" %} class="selected"{% endif %}><a href="{% url 'uploader' %}"><span class="material-symbols-outlined">cloud_upload</span>Upload Data</a></li>
                     <li><a href="https://postgresml.org/user_guides/training/overview/" data-turbo="false" target="_blank"><span class="material-symbols-outlined">menu_book</span>Docs</a></li>
                 </ul>
             </nav>
diff --git a/pgml-dashboard/app/templates/uploader/index.html b/pgml-dashboard/app/templates/uploader/index.html
@@ -0,0 +1,52 @@
+{% extends "base.html" %}
+{% load humanize %}
+
+{% block main %}
+
+<section>
+  {% if error %}
+  <h1><span class="material-symbols-outlined" style="color: var(--highlite-red)">cloud_upload</span>Error</h1>
+  {% else %}
+  <h1><span class="material-symbols-outlined">cloud_upload</span>Upload Data</h1>
+  {% endif %}
+  
+  {% if error %}
+    <p style="margin-bottom: 1rem;">Hmm, something went wrong. Please make sure:</p>
+  {% else %}
+    <p style="margin-bottom: 1rem;">You can upload your datasets using the CSV format. Before uploading, please make sure:</p>
+  {% endif %}
+  
+  <ol>
+    <li>The data is numeric (i.e. only floats or integers and no text)</li>
+    <li>The CSV includes headers on the first line</li>
+    <li>The headers are alphanumeric, contain no spaces and don't start with a number</li>
+    <li>The CSV is comma (<code>,</code>) delimited</li>
+  </ol>
+
+  {% if error %}
+  <h4>Error: </h4>
+  <div class="markdown-body">
+    <pre><code>{{ error }}</code></pre>
+  </div>
+  {% endif %}
+
+  <p>If you are exporting data from a PostgreSQL database, you can use <code>psql</code> to generate a valid CSV file:</p>
+  <div class="markdown-body">
+    <pre><code class="language-sql">\copy your_table_name TO 'output.csv' CSV HEADER</code></pre>
+  </div>
+
+</section>
+
+<section>
+  <form action="{% url 'uploader' %}" method="post" enctype="multipart/form-data">
+    {% csrf_token %}
+    <div class="flex">
+      <input id="file" type="file" name="file" accept="text/csv,application/json" required="true" />
+    </div>
+
+    <div class="button-container">
+      <button type="submit">Upload</button>
+    </div>
+  </form>
+</section>
+{% endblock %}
diff --git a/pgml-dashboard/app/templates/uploader/uploaded.html b/pgml-dashboard/app/templates/uploader/uploaded.html
@@ -0,0 +1,25 @@
+{% extends "base.html" %}
+{% load humanize %}
+
+{% block main %}
+
+<section>
+  <h1><span class="material-symbols-outlined" style="color: var(--highlite-green)">cloud_upload</span>Upload Successful</h1>
+</section>
+
+<section>
+  <h2><span class="material-symbols-outlined">data_array</span>Preview</h2>
+  
+  {% include 'projects/sample.html' %}
+</section>
+
+<section>
+  <h2><span class="material-symbols-outlined">table_rows</span>Next Steps</h2>
+  <p>Your data has been saved in <strong>pgml.{{ table_name }}</strong> table.</p>
+  <p>You can now build a model using a <a href="{% url 'notebooks' %}">Notebook</a> or browse the data in the <a href="{% url 'console' %}">Console</a>:</p>
+  <div class="markdown-body">
+    <pre><code class="language-sql">SELECT * FROM pgml.{{ table_name }}
+LIMIT 10</code></pre>
+</section>
+
+{% endblock %}
diff --git a/pgml-dashboard/app/urls.py b/pgml-dashboard/app/urls.py
@@ -1,7 +1,7 @@
 from django.urls import path
 from rest_framework import routers
 
-from app.views import root, projects, models, snapshots, deployments, console, notebooks
+from app.views import root, projects, models, snapshots, deployments, console, notebooks, uploader
 
 
 router = routers.DefaultRouter()
@@ -26,6 +26,8 @@
     path("projects/<int:pk>", projects.ProjectView.as_view(), name="project"),
     path("snapshots/", snapshots.index, name="snapshots"),
     path("snapshots/<int:id>", snapshots.snapshot, name="snapshot"),
+    path("uploader/", uploader.index, name="uploader"),
+    path("uploader/uploaded/<int:pk>/", uploader.uploaded, name="uploader/uploaded"),
     path("console/", console.ConsoleView.as_view(), name="console"),
     path("console/run/", console.run_sql, name="console/run-sql"),
     path("set-auth-cookie/", root.set_auth_cookie, name="set-auth-cookie"),
diff --git a/pgml-dashboard/app/views/uploader.py b/pgml-dashboard/app/views/uploader.py
@@ -0,0 +1,72 @@
+from django.db import connection
+from django.shortcuts import render, get_object_or_404
+from django.utils.safestring import SafeString
+from django.http import HttpResponse, HttpResponseRedirect
+from django.urls import reverse_lazy
+from django import forms
+
+from app.models import UploadedData
+
+import csv
+import json
+import codecs
+
+
+class UploadForm(forms.Form):
+    file = forms.FileField()
+
+
+def index(request):
+    if request.method == "POST":
+        form = UploadForm(request.POST, request.FILES)
+        if not form.is_valid():
+            return HttpResponse(status=400)
+
+        file = request.FILES.get("file")
+        if file.content_type not in ["text/csv", "application/json"]:
+            return HttpResponse(status=400)
+        else:
+            try:
+                upload = UploadedData.objects.create(
+                    file_type=1 if file.content_type == "text/csv" else 2,
+                )
+
+                upload.create_table(file)
+            except Exception as e:
+                return render(
+                    request,
+                    "uploader/index.html",
+                    {
+                        "error": str(e),
+                        "topic": "uploader",
+                    },
+                    status=400,
+                )
+            return HttpResponseRedirect(reverse_lazy("uploader/uploaded", kwargs={"pk": upload.pk}))
+    else:
+        return render(
+            request,
+            "uploader/index.html",
+            {
+                "topic": "uploader",
+            },
+        )
+
+
+def uploaded(request, pk):
+    upload = UploadedData.objects.get(pk=pk)
+    with connection.cursor() as cursor:
+        cursor.execute(f"SELECT * FROM data_{upload.pk} LIMIT 11")
+        columns = [col[0] for col in cursor.description]
+        rows = cursor.fetchall()
+    return render(
+        request,
+        "uploader/uploaded.html",
+        {
+            "columns": columns,
+            "rows": rows[:10],
+            "table_name": f"data_{upload.pk}",
+            "redacted": len(rows) > 10,
+            "topic": "uploader",
+        },
+    )
