From a4fca261371f508721e45700e39b950e85b6718a Mon Sep 17 00:00:00 2001
From: erikamenezes <menezes.erika90@gmail.com>
Date: Mon, 20 May 2019 22:01:09 +0000
Subject: [PATCH 1/9] added feature to export jsaon text labels

---
 app/server/api.py                             | 12 ++++++++--
 .../static/js/download_sequence_labeling.vue  | 18 +++++++++++++++
 .../download_sequence_labeling.json1l         |  3 +++
 app/server/utils.py                           | 22 +++++++++++++++++--
 4 files changed, 51 insertions(+), 4 deletions(-)
 create mode 100644 app/server/static/js/examples/download_sequence_labeling.json1l

diff --git a/app/server/api.py b/app/server/api.py
index bc853c4d..5376f7de 100644
--- a/app/server/api.py
+++ b/app/server/api.py
@@ -209,13 +209,21 @@ class TextDownloadAPI(APIView):
         project = get_object_or_404(Project, pk=self.kwargs['project_id'])
         documents = project.documents.all()
         painter = self.select_painter(format)
-        data = painter.paint(documents)
+        
+        # json1 format prints text labels while json format prints annotations with label ids
+		# json1 format - "labels": [[0, 15, "PERSON"], ..]
+		# json format  "annotations": [{"label": 5, "start_offset": 0, "end_offset": 2, "user": 1},..]
+        if format == "json1":
+            labels = project.labels.all()
+            data = painter.paint_labels(documents, labels)
+        else:
+            data = painter.paint(documents)
         return Response(data)
 
     def select_painter(self, format):
         if format == 'csv':
             return CSVPainter()
-        elif format == 'json':
+        elif format == 'json' or format == "json1":
             return JSONPainter()
         else:
             raise ValidationError('format {} is invalid.'.format(format))
diff --git a/app/server/static/js/download_sequence_labeling.vue b/app/server/static/js/download_sequence_labeling.vue
index 10529fac..dfea48a7 100644
--- a/app/server/static/js/download_sequence_labeling.vue
+++ b/app/server/static/js/download_sequence_labeling.vue
@@ -12,11 +12,29 @@ block select-format-area
     )
     | JSONL
 
+  label.radio
+    input(
+      type="radio"
+      name="format"
+      value="json1"
+      v-bind:checked="format == 'json1'"
+      v-model="format"
+    )
+    | JSON(Text-Labels)
+
+
 block example-format-area
   pre.code-block(v-show="format == 'json'")
     code.json
       include ./examples/download_sequence_labeling.jsonl
       | ...
+  pre.code-block(v-show="format == 'json1'")
+    code.json
+      include ./examples/download_sequence_labeling.json1l
+      | ...
+</template>
+
+
 </template>
 
 <script>
diff --git a/app/server/static/js/examples/download_sequence_labeling.json1l b/app/server/static/js/examples/download_sequence_labeling.json1l
new file mode 100644
index 00000000..cdf3e1fb
--- /dev/null
+++ b/app/server/static/js/examples/download_sequence_labeling.json1l
@@ -0,0 +1,3 @@
+{"id": 1, "text": "EU rejects ...", "labels": [[0,2,"ORG"], [11,17, "MISC"], [34,41,"ORG"]]}
+{"id": 2, "text": "Peter Blackburn", "labels": [[0, 15, "PERSON"]]}
+{"id": 3, "text": "President Obama", "labels": [[10, 15, "PERSON"]]}
diff --git a/app/server/utils.py b/app/server/utils.py
index e33bbed5..d0a638f0 100644
--- a/app/server/utils.py
+++ b/app/server/utils.py
@@ -349,7 +349,8 @@ class JSONParser(FileParser):
                 yield data
                 data = []
             try:
-                j = json.loads(line)
+                #j = json.loads(line)
+                j  = json.loads(line.decode('utf-8'))
                 j['meta'] = json.dumps(j.get('meta', {}))
                 data.append(j)
             except json.decoder.JSONDecodeError:
@@ -376,7 +377,6 @@ class JSONLRenderer(JSONRenderer):
                              ensure_ascii=self.ensure_ascii,
                              allow_nan=not self.strict) + '\n'
 
-
 class JSONPainter(object):
 
     def paint(self, documents):
@@ -391,6 +391,24 @@ class JSONPainter(object):
             data.append(d)
         return data
 
+    def paint_labels(self, documents, labels):
+        serializer_labels = LabelSerializer(labels, many = True)
+        serializer = DocumentSerializer(documents, many=True)
+        data = []
+        for d in serializer.data:
+            labels = []
+            for a in d['annotations']:
+                label_obj = [x for x in serializer_labels.data if x['id'] == a['label']][0]
+                label_text = label_obj['text']
+                label_start = a['start_offset']
+                label_end = a['end_offset']
+                labels. append([label_start, label_end, label_text])
+            d.pop('annotations')
+            d['labels'] = labels
+            d['meta'] = json.loads(d['meta'])
+            data.append(d)
+        return data
+
 
 class CSVPainter(JSONPainter):
 

From 2a0cc3566785b79ef27a60ffd5474361c86c7ec9 Mon Sep 17 00:00:00 2001
From: erikamenezes <menezes.erika90@gmail.com>
Date: Mon, 20 May 2019 22:20:36 +0000
Subject: [PATCH 2/9] minor changes

---
 app/server/api.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/server/api.py b/app/server/api.py
index 5376f7de..791f0777 100644
--- a/app/server/api.py
+++ b/app/server/api.py
@@ -211,8 +211,8 @@ class TextDownloadAPI(APIView):
         painter = self.select_painter(format)
         
         # json1 format prints text labels while json format prints annotations with label ids
-		# json1 format - "labels": [[0, 15, "PERSON"], ..]
-		# json format  "annotations": [{"label": 5, "start_offset": 0, "end_offset": 2, "user": 1},..]
+        # json1 format - "labels": [[0, 15, "PERSON"], ..]
+        # json format - "annotations": [{"label": 5, "start_offset": 0, "end_offset": 2, "user": 1},..]
         if format == "json1":
             labels = project.labels.all()
             data = painter.paint_labels(documents, labels)

From cf934661451dc1a76958a0fbfe7f615e83eecca3 Mon Sep 17 00:00:00 2001
From: erikamenezes <menezes.erika90@gmail.com>
Date: Mon, 20 May 2019 22:36:28 +0000
Subject: [PATCH 3/9] removing empty lines

---
 app/server/static/js/download_sequence_labeling.vue | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/app/server/static/js/download_sequence_labeling.vue b/app/server/static/js/download_sequence_labeling.vue
index dfea48a7..e6793a09 100644
--- a/app/server/static/js/download_sequence_labeling.vue
+++ b/app/server/static/js/download_sequence_labeling.vue
@@ -22,7 +22,6 @@ block select-format-area
     )
     | JSON(Text-Labels)
 
-
 block example-format-area
   pre.code-block(v-show="format == 'json'")
     code.json
@@ -34,9 +33,6 @@ block example-format-area
       | ...
 </template>
 
-
-</template>
-
 <script>
 import { uploadMixin } from './mixin';
 

From 2d702474fbfc84f2af6007e3e6b86a5712e7afcf Mon Sep 17 00:00:00 2001
From: erikamenezes <menezes.erika90@gmail.com>
Date: Tue, 21 May 2019 19:57:01 +0000
Subject: [PATCH 4/9] adding test for downloading json format with text labels

---
 app/server/tests/test_api.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/app/server/tests/test_api.py b/app/server/tests/test_api.py
index 3177b5c9..ba3bfff1 100644
--- a/app/server/tests/test_api.py
+++ b/app/server/tests/test_api.py
@@ -904,6 +904,11 @@ class TestDownloader(APITestCase):
                                   format='json',
                                   expected_status=status.HTTP_200_OK)
 
+    def test_can_download_labelling_json1(self):
+        self.download_test_helper(url=self.labeling_url,
+                                  format='json1',
+                                  expected_status=status.HTTP_200_OK)    
+
     def test_can_download_plain_text(self):
         self.download_test_helper(url=self.classification_url,
                                   format='plain',

From 8e246eca2834f6c99c605cba834731b27425a0a2 Mon Sep 17 00:00:00 2001
From: erikamenezes <menezes.erika90@gmail.com>
Date: Tue, 21 May 2019 20:14:55 +0000
Subject: [PATCH 5/9] style change to vue file

---
 app/server/static/js/download_sequence_labeling.vue | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/server/static/js/download_sequence_labeling.vue b/app/server/static/js/download_sequence_labeling.vue
index e6793a09..109dc96d 100644
--- a/app/server/static/js/download_sequence_labeling.vue
+++ b/app/server/static/js/download_sequence_labeling.vue
@@ -27,6 +27,7 @@ block example-format-area
     code.json
       include ./examples/download_sequence_labeling.jsonl
       | ...
+
   pre.code-block(v-show="format == 'json1'")
     code.json
       include ./examples/download_sequence_labeling.json1l

From 8ffab2c4f17f16ecb2f30fc52fa1bb96b73b9429 Mon Sep 17 00:00:00 2001
From: erikamenezes <menezes.erika90@gmail.com>
Date: Wed, 22 May 2019 19:45:33 +0000
Subject: [PATCH 6/9] addressing review comments, TextIOWrapper for UTF-8
 decoding

---
 app/server/utils.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/app/server/utils.py b/app/server/utils.py
index d0a638f0..4d8f5ea5 100644
--- a/app/server/utils.py
+++ b/app/server/utils.py
@@ -343,14 +343,15 @@ class CSVParser(FileParser):
 class JSONParser(FileParser):
 
     def parse(self, file):
+        file = io.TextIOWrapper(file, encoding='utf-8')
         data = []
         for i, line in enumerate(file, start=1):
             if len(data) >= IMPORT_BATCH_SIZE:
                 yield data
                 data = []
             try:
-                #j = json.loads(line)
-                j  = json.loads(line.decode('utf-8'))
+                j = json.loads(line)
+                #j  = json.loads(line.decode('utf-8'))
                 j['meta'] = json.dumps(j.get('meta', {}))
                 data.append(j)
             except json.decoder.JSONDecodeError:
@@ -392,7 +393,7 @@ class JSONPainter(object):
         return data
 
     def paint_labels(self, documents, labels):
-        serializer_labels = LabelSerializer(labels, many = True)
+        serializer_labels = LabelSerializer(labels, many=True)
         serializer = DocumentSerializer(documents, many=True)
         data = []
         for d in serializer.data:
@@ -402,7 +403,7 @@ class JSONPainter(object):
                 label_text = label_obj['text']
                 label_start = a['start_offset']
                 label_end = a['end_offset']
-                labels. append([label_start, label_end, label_text])
+                labels.append([label_start, label_end, label_text])
             d.pop('annotations')
             d['labels'] = labels
             d['meta'] = json.loads(d['meta'])

From 2ada3e62480d344a6ab4c1dab05c41d1086b3a02 Mon Sep 17 00:00:00 2001
From: erikamenezes <menezes.erika90@gmail.com>
Date: Fri, 24 May 2019 16:19:58 +0000
Subject: [PATCH 7/9] Codacy code quality improvements

---
 app/server/api.py            | 1 -
 app/server/tests/test_api.py | 2 +-
 app/server/utils.py          | 2 +-
 3 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/app/server/api.py b/app/server/api.py
index 791f0777..595aa46c 100644
--- a/app/server/api.py
+++ b/app/server/api.py
@@ -209,7 +209,6 @@ class TextDownloadAPI(APIView):
         project = get_object_or_404(Project, pk=self.kwargs['project_id'])
         documents = project.documents.all()
         painter = self.select_painter(format)
-        
         # json1 format prints text labels while json format prints annotations with label ids
         # json1 format - "labels": [[0, 15, "PERSON"], ..]
         # json format - "annotations": [{"label": 5, "start_offset": 0, "end_offset": 2, "user": 1},..]
diff --git a/app/server/tests/test_api.py b/app/server/tests/test_api.py
index ba3bfff1..a570183c 100644
--- a/app/server/tests/test_api.py
+++ b/app/server/tests/test_api.py
@@ -907,7 +907,7 @@ class TestDownloader(APITestCase):
     def test_can_download_labelling_json1(self):
         self.download_test_helper(url=self.labeling_url,
                                   format='json1',
-                                  expected_status=status.HTTP_200_OK)    
+                                  expected_status=status.HTTP_200_OK)
 
     def test_can_download_plain_text(self):
         self.download_test_helper(url=self.classification_url,
diff --git a/app/server/utils.py b/app/server/utils.py
index 4d8f5ea5..0f98830f 100644
--- a/app/server/utils.py
+++ b/app/server/utils.py
@@ -392,6 +392,7 @@ class JSONPainter(object):
             data.append(d)
         return data
 
+    @staticmethod
     def paint_labels(self, documents, labels):
         serializer_labels = LabelSerializer(labels, many=True)
         serializer = DocumentSerializer(documents, many=True)
@@ -410,7 +411,6 @@ class JSONPainter(object):
             data.append(d)
         return data
 
-
 class CSVPainter(JSONPainter):
 
     def paint(self, documents):

From 603388ead407036528cdf8f01fb659ea599882e8 Mon Sep 17 00:00:00 2001
From: erikamenezes <menezes.erika90@gmail.com>
Date: Fri, 24 May 2019 16:43:53 +0000
Subject: [PATCH 8/9] rmeoving additional argument

---
 app/server/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/server/utils.py b/app/server/utils.py
index 0f98830f..ad92a924 100644
--- a/app/server/utils.py
+++ b/app/server/utils.py
@@ -393,7 +393,7 @@ class JSONPainter(object):
         return data
 
     @staticmethod
-    def paint_labels(self, documents, labels):
+    def paint_labels(documents, labels):
         serializer_labels = LabelSerializer(labels, many=True)
         serializer = DocumentSerializer(documents, many=True)
         data = []

From 0c004a9bec5bf1ee53195a5dd8a0b7e91bd18ba4 Mon Sep 17 00:00:00 2001
From: erikamenezes <menezes.erika90@gmail.com>
Date: Fri, 24 May 2019 17:03:17 +0000
Subject: [PATCH 9/9] moving example + adding static method

---
 app/server/api.py                                               | 2 +-
 .../examples/download_sequence_labeling.json1l                  | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename app/server/static/{js => components}/examples/download_sequence_labeling.json1l (100%)

diff --git a/app/server/api.py b/app/server/api.py
index 595aa46c..9df78df8 100644
--- a/app/server/api.py
+++ b/app/server/api.py
@@ -214,7 +214,7 @@ class TextDownloadAPI(APIView):
         # json format - "annotations": [{"label": 5, "start_offset": 0, "end_offset": 2, "user": 1},..]
         if format == "json1":
             labels = project.labels.all()
-            data = painter.paint_labels(documents, labels)
+            data = JSONPainter.paint_labels(documents, labels)
         else:
             data = painter.paint(documents)
         return Response(data)
diff --git a/app/server/static/js/examples/download_sequence_labeling.json1l b/app/server/static/components/examples/download_sequence_labeling.json1l
similarity index 100%
rename from app/server/static/js/examples/download_sequence_labeling.json1l
rename to app/server/static/components/examples/download_sequence_labeling.json1l