From 4312843f3507c9fe43b4c809d1babd12b6b6dba1 Mon Sep 17 00:00:00 2001
From: Bramble Xu <liangxu006@gmail.com>
Date: Tue, 27 Nov 2018 14:32:07 +0900
Subject: [PATCH] iss17: support for uploading json file

---
 .../templates/admin/dataset_upload.html       |  7 ++--
 app/server/tests/test_views.py                | 25 ++++++++++++--
 app/server/views.py                           | 33 +++++++++++++------
 3 files changed, 49 insertions(+), 16 deletions(-)

diff --git a/app/server/templates/admin/dataset_upload.html b/app/server/templates/admin/dataset_upload.html
index e2cd97b1..ca101151 100644
--- a/app/server/templates/admin/dataset_upload.html
+++ b/app/server/templates/admin/dataset_upload.html
@@ -27,11 +27,12 @@
           <div class="section">
             <div class="control">
               <label class="radio mb10">
-                <input type="radio" name="format" checked> Upload a TXT file from your computer
-              </label>
+                <input type="radio" name="format" value="csv" checked> Upload a TXT file from your computer<br>
+                <input type="radio" name="format" value="json" checked> Upload a JSON file from your computer<br>
+               </label>
               <div class="file has-name is-small mb20">
                 <label class="file-label">
-                  <input class="file-input" type="file" ref="file" name="csv_file" required v-on:change="handleFileUpload()">
+                  <input class="file-input" type="file" ref="file" name="file" required v-on:change="handleFileUpload()">
                   <span class="file-cta">
                     <span class="file-icon">
                       <i class="fas fa-upload"></i>
diff --git a/app/server/tests/test_views.py b/app/server/tests/test_views.py
index faccd65e..79fd7df3 100644
--- a/app/server/tests/test_views.py
+++ b/app/server/tests/test_views.py
@@ -12,6 +12,8 @@ class TestUpload(TestCase):
         self.username, self.password = 'user', 'pass'
         self.client = Client()
         self.filepath = os.path.join(os.path.dirname(__file__), 'data/test.csv')
+        self.jsonpath = os.path.join(os.path.dirname(__file__), 'data/test.jsonl')
+
 
     def create_user(self):
         user = User.objects.create_user(username=self.username, password=self.password)
@@ -40,7 +42,7 @@ class TestUpload(TestCase):
         with open(self.filepath) as f:
             url = reverse('upload', args=[project.id])
             self.client.login(username=self.username, password=self.password)
-            response = self.client.post(url, {'csv_file': f})
+            response = self.client.post(url, {'file': f, 'format': 'csv'})
 
         self.assertEqual(response.status_code, status.HTTP_302_FOUND)
         self.assertEqual(Document.objects.count(), 3)
@@ -56,7 +58,24 @@ class TestUpload(TestCase):
         with open(self.filepath) as f:
             url = reverse('upload', args=[project.id])
             self.client.login(username=self.username, password=self.password)
-            response = self.client.post(url, {'csv_file': f})
+            response = self.client.post(url, {'file': f, 'format': 'csv'})
 
-        self.assertEqual(response.status_code, status.HTTP_302_FOUND)
+        self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
         self.assertEqual(Document.objects.count(), 0)
+
+    def test_upload_json_by_admin(self):
+        """
+        Ensure we can create a new document object from json by admin.
+        """
+        user = self.create_superuser()
+        project = self.create_project()
+        project.users.add(user)
+
+        with open(self.jsonpath, 'r') as f:
+            url = reverse('upload', args=[project.id])
+            self.client.login(username=self.username, password=self.password)
+            response = self.client.post(url, {'file': f, 'format': 'json'})
+
+        print(self.jsonpath)
+        self.assertEqual(response.status_code, status.HTTP_302_FOUND)
+        self.assertEqual(Document.objects.count(), 18)
\ No newline at end of file
diff --git a/app/server/views.py b/app/server/views.py
index 8b7bc58a..afdec3ce 100644
--- a/app/server/views.py
+++ b/app/server/views.py
@@ -1,4 +1,5 @@
 import csv
+import json
 from io import TextIOWrapper
 
 from django.urls import reverse
@@ -56,17 +57,29 @@ class DataUpload(SuperUserMixin, LoginRequiredMixin, TemplateView):
 
     def post(self, request, *args, **kwargs):
         project = get_object_or_404(Project, pk=kwargs.get('project_id'))
+        import_format = request.POST['format']
         try:
-            form_data = TextIOWrapper(request.FILES['csv_file'].file, encoding='utf-8')
-            if project.is_type_of(Project.SEQUENCE_LABELING):
-                Document.objects.bulk_create([Document(
-                    text=line.strip(),
-                    project=project) for line in form_data])
-            else:
-                reader = csv.reader(form_data)
-                Document.objects.bulk_create([Document(
-                    text=line[0].strip(),
-                    project=project) for line in reader])
+            if import_format == 'csv':
+                form_data = TextIOWrapper(
+                    request.FILES['file'].file, encoding='utf-8')
+                if project.is_type_of(Project.SEQUENCE_LABELING):
+                    Document.objects.bulk_create([
+                        Document(text=line.strip(), project=project)
+                        for line in form_data
+                    ])
+                else:
+                    reader = csv.reader(form_data)
+                    Document.objects.bulk_create([
+                        Document(text=line[0].strip(), project=project)
+                        for line in reader
+                    ])
+
+            elif import_format == 'json':
+                form_data = request.FILES['file'].file
+                Document.objects.bulk_create([
+                    Document(text=json.loads(entry)['text'], project=project)
+                    for entry in form_data
+                ])
             return HttpResponseRedirect(reverse('dataset', args=[project.id]))
         except:
             return HttpResponseRedirect(reverse('upload', args=[project.id]))