Allow CSVs with no headers (#310)

levkk · web-flow · commit a5a1ff8d51b9 · 2022-09-09T12:22:07.000-07:00
diff --git a/pgml-dashboard/app/models.py b/pgml-dashboard/app/models.py
@@ -318,17 +318,20 @@ class UploadedData(models.Model):
     created_at = models.DateTimeField(auto_now_add=True)
     updated_at = models.DateTimeField(auto_now=True)
 
-    def create_table(self, file):
+    def create_table(self, file, has_header=False):
         if file.content_type == "text/csv":
             reader = csv.reader(codecs.iterdecode(file, "utf-8"))
             headers = next(reader)
-            columns = ", ".join(map(lambda x: f"{x.replace(' ', '_').lower()} FLOAT4", headers))
+
+            if has_header:
+                columns = ", ".join(map(lambda x: f"{x.replace(' ', '_').lower()} TEXT", headers))
+            else:
+                columns = ", ".join(map(lambda x: f"column_{x} TEXT", range(len(headers))))
 
             with transaction.atomic():
                 sql = f"CREATE TABLE data_{self.pk} (" + columns + ")"
 
                 with connection.cursor() as cursor:
                     cursor.execute(sql)
-
                     file.seek(0)
-                    cursor.copy_expert(f"COPY data_{self.pk} FROM STDIN CSV HEADER", file)
+                    cursor.copy_expert(f"COPY data_{self.pk} FROM STDIN CSV {'HEADER' if has_header else ''}", file)
diff --git a/pgml-dashboard/app/static/css/base.css b/pgml-dashboard/app/static/css/base.css
@@ -685,3 +685,39 @@ body.uploader section li {
 body.uploader strong {
   font-weight: bold;
 }
+
+body.uploader label {
+  user-select: none;
+  cursor: pointer;
+}
+
+/*
+ * Checkbox
+ */
+input[type=checkbox] {
+  /* Reset style */
+  appearance: none;
+
+  background: transparent;
+  border: 1px solid var(--gray-5);
+  
+  height: 1.6em;
+  width: 1.6em;
+  
+  border-radius: 3px;
+  
+  display: inline-flex;
+  align-items: center;
+  justify-content: center;
+  position: relative;
+
+  cursor: pointer;
+}
+
+input[type=checkbox]:checked:after {
+  content: '\2714';
+  font-size: 1em;
+  position: absolute;
+  color: var(--highlite-green);
+  filter:  brightness(0.75);
+}
diff --git a/pgml-dashboard/app/templates/uploader/index.html b/pgml-dashboard/app/templates/uploader/index.html
@@ -11,15 +11,13 @@ <h1><span class="material-symbols-outlined">cloud_upload</span>Upload Data</h1>
   {% endif %}
   
   {% if error %}
-    <p style="margin-bottom: 1rem;">Hmm, something went wrong. Please make sure:</p>
+    <p>Hmm, something went wrong. Make sure:</p>
   {% else %}
-    <p style="margin-bottom: 1rem;">You can upload your datasets using the CSV format. Before uploading, please make sure:</p>
+    <p>You can upload your datasets using the CSV format. Before uploading, make sure:</p>
   {% endif %}
   
   <ol>
-    <li>The data is numeric (i.e. only floats or integers and no text)</li>
-    <li>The CSV includes headers on the first line</li>
-    <li>The headers are alphanumeric, contain no spaces and don't start with a number</li>
+    <li>If the CSV has a header, the column names are alphanumeric, contain no spaces and don't start with a number</li>
     <li>The CSV is comma (<code>,</code>) delimited</li>
   </ol>
 
@@ -30,7 +28,7 @@ <h4>Error: </h4>
   </div>
   {% endif %}
 
-  <p>If you are exporting data from a PostgreSQL database, you can use <code>psql</code> to generate a valid CSV file:</p>
+  <p>If you are exporting data from a PostgreSQL database, you can use <code>psql</code> to generate a valid CSV file with a header:</p>
   <div class="markdown-body">
     <pre><code class="language-sql">\copy your_table_name TO 'output.csv' CSV HEADER</code></pre>
   </div>
@@ -40,8 +38,12 @@ <h4>Error: </h4>
 <section>
   <form action="{% url 'uploader' %}" method="post" enctype="multipart/form-data">
     {% csrf_token %}
-    <div class="flex">
+    <div class="flex flex-center">
       <input id="file" type="file" name="file" accept="text/csv,application/json" required="true" />
+      <div class="flex flex-center" style="margin-left: 2rem;">
+        <input type="checkbox" name="has_header" checked id="has_header" style="margin-right: 0.5rem;" checked />
+        <label for="has_header">My CSV has a header</label>
+      </div>
     </div>
 
     <div class="button-container">
diff --git a/pgml-dashboard/app/templates/uploader/uploaded.html b/pgml-dashboard/app/templates/uploader/uploaded.html
@@ -15,10 +15,21 @@ <h2><span class="material-symbols-outlined">data_array</span>Preview</h2>
 
 <section>
   <h2><span class="material-symbols-outlined">table_rows</span>Next Steps</h2>
-  <p>Your data has been saved in <strong>pgml.{{ table_name }}</strong> table.</p>
-  <p>You can now build a model using a <a href="{% url 'notebooks' %}">Notebook</a> or browse the data in the <a href="{% url 'console' %}">Console</a>:</p>
+  <p>Your data has been saved in <strong>pgml.{{ table_name }}</strong> table. You can explore the data in the <a href="{% url 'console' %}">Console</a>:</p>
   <div class="markdown-body">
     <pre><code class="language-sql">SELECT * FROM pgml.{{ table_name }}
+LIMIT 10</code></pre>
+  </div>
+<p> All columns were converted to text, so you'll need to cast them to the appropriate data type before training a model, for example:</p>
+  <div class="markdown-body">
+    <pre><code class="language-sql">CREATE MATERIALIZED VIEW pgml.{{ table_name }}_view AS
+SELECT {% for column in columns %}
+  CAST({{ column }} AS FLOAT4){% if not forloop.last %},{% else %}
+{% endif %}{% endfor %}FROM pgml.{{ table_name }}</code></pre>
+  </div>
+  <p>You can now run experiments and build models using <a href="{% url 'notebooks' %}">Notebooks</a>:</p>
+  <div class="markdown-body">
+    <pre><code class="language-sql">SELECT * FROM pgml.{{ table_name }}_view
 LIMIT 10</code></pre>
 </section>
 
diff --git a/pgml-dashboard/app/views/console.py b/pgml-dashboard/app/views/console.py
@@ -48,16 +48,20 @@ def run_sql(request):
         try:
             cursor.execute("SET statement_timeout = '30s'")
             cursor.execute(query)
-            results = cursor.fetchall()
 
-            return render(
-                request,
-                "projects/sample.html",
-                {
-                    "columns": [desc[0] for desc in cursor.description],
-                    "rows": results,
-                },
-            )
+            if cursor.description:
+                results = cursor.fetchall()
+
+                return render(
+                    request,
+                    "projects/sample.html",
+                    {
+                        "columns": [desc[0] for desc in cursor.description],
+                        "rows": results,
+                    },
+                )
+            else:
+                raise Exception(str(cursor.statusmessage))
         except Exception as e:
             return HttpResponse(
                 f"""
diff --git a/pgml-dashboard/app/views/uploader.py b/pgml-dashboard/app/views/uploader.py
@@ -14,6 +14,7 @@
 
 class UploadForm(forms.Form):
     file = forms.FileField()
+    has_header = forms.BooleanField(required=False)
 
 
 def index(request):
@@ -31,7 +32,7 @@ def index(request):
                     file_type=1 if file.content_type == "text/csv" else 2,
                 )
 
-                upload.create_table(file)
+                upload.create_table(file, form.cleaned_data.get("has_header", False))
             except Exception as e:
                 return render(
                     request,
