diff --git a/backend/api/admin.py b/backend/api/admin.py index 09c661e3..bd4a1dfd 100644 --- a/backend/api/admin.py +++ b/backend/api/admin.py @@ -1,6 +1,6 @@ from django.contrib import admin -from .models import (Category, Comment, Document, Label, Project, Role, +from .models import (Category, Comment, Example, Label, Project, Role, RoleMapping, Seq2seqProject, SequenceLabelingProject, Span, Tag, TextClassificationProject, TextLabel) @@ -11,7 +11,7 @@ class LabelAdmin(admin.ModelAdmin): search_fields = ('text',) -class DocumentAdmin(admin.ModelAdmin): +class ExampleAdmin(admin.ModelAdmin): list_display = ('text', 'project', 'meta') ordering = ('project',) search_fields = ('text',) @@ -66,7 +66,7 @@ admin.site.register(Category, CategoryAdmin) admin.site.register(Span, SpanAdmin) admin.site.register(TextLabel, TextLabelAdmin) admin.site.register(Label, LabelAdmin) -admin.site.register(Document, DocumentAdmin) +admin.site.register(Example, ExampleAdmin) admin.site.register(Project, ProjectAdmin) admin.site.register(TextClassificationProject, ProjectAdmin) admin.site.register(SequenceLabelingProject, ProjectAdmin) diff --git a/backend/api/filters.py b/backend/api/filters.py index 4e90b148..8dc1f484 100644 --- a/backend/api/filters.py +++ b/backend/api/filters.py @@ -1,7 +1,7 @@ from django.db.models import Count, Q from django_filters.rest_framework import BooleanFilter, FilterSet -from .models import Document +from .models import Example class DocumentFilter(FilterSet): @@ -25,7 +25,7 @@ class DocumentFilter(FilterSet): return queryset class Meta: - model = Document + model = Example fields = ( 'project', 'text', 'created_at', 'updated_at', 'categories__label__id', 'spans__label__id', diff --git a/backend/api/models.py b/backend/api/models.py index e64bc22c..6006dd72 100644 --- a/backend/api/models.py +++ b/backend/api/models.py @@ -126,7 +126,7 @@ class Label(models.Model): ) -class Example(PolymorphicModel): +class Example(models.Model): meta = models.JSONField(default=dict) filename = models.FileField(default='.') project = models.ForeignKey( @@ -140,6 +140,7 @@ class Example(PolymorphicModel): null=True, blank=True ) + text = models.TextField(null=True, blank=True) created_at = models.DateTimeField(auto_now_add=True) updated_at = models.DateTimeField(auto_now=True) @@ -148,19 +149,6 @@ class Example(PolymorphicModel): return Comment.objects.filter(example=self.id).count() -class Document(Example): - text = models.TextField() - - def __str__(self): - return self.text[:50] - - -class Image(Example): - - def __str__(self): - return self.filename - - class Comment(models.Model): text = models.TextField() example = models.ForeignKey( diff --git a/backend/api/serializers.py b/backend/api/serializers.py index 8fc7c93f..ff591da6 100644 --- a/backend/api/serializers.py +++ b/backend/api/serializers.py @@ -8,10 +8,10 @@ from rest_polymorphic.serializers import PolymorphicSerializer from .models import (DOCUMENT_CLASSIFICATION, SEQ2SEQ, SEQUENCE_LABELING, SPEECH2TEXT, AutoLabelingConfig, Category, Comment, - Document, Example, Image, ImageClassificationProject, - Label, Project, Role, RoleMapping, Seq2seqProject, - SequenceLabelingProject, Span, Speech2textProject, Tag, - TextClassificationProject, TextLabel) + Example, ImageClassificationProject, Label, Project, Role, + RoleMapping, Seq2seqProject, SequenceLabelingProject, + Span, Speech2textProject, Tag, TextClassificationProject, + TextLabel) class UserSerializer(serializers.ModelSerializer): @@ -78,7 +78,7 @@ class TagSerializer(serializers.ModelSerializer): read_only_fields = ('id', 'project') -class BaseDataSerializer(serializers.ModelSerializer): +class ExampleSerializer(serializers.ModelSerializer): annotations = serializers.SerializerMethodField() annotation_approver = serializers.SerializerMethodField() @@ -100,36 +100,22 @@ class BaseDataSerializer(serializers.ModelSerializer): class Meta: model = Example - fields = ['id', 'filename', 'annotations', 'meta', 'annotation_approver', 'comment_count'] + fields = [ + 'id', + 'filename', + 'annotations', + 'meta', + 'annotation_approver', + 'comment_count', + 'text' + ] read_only_fields = ['filename'] -class DocumentSerializer(BaseDataSerializer): - - class Meta: - model = Document - fields = BaseDataSerializer.Meta.fields + ['text'] - - -class ImageSerializer(BaseDataSerializer): - - class Meta: - model = Image - fields = BaseDataSerializer.Meta.fields - - -class ExampleSerializer(PolymorphicSerializer): - model_serializer_mapping = { - Example: BaseDataSerializer, - Document: DocumentSerializer, - Image: ImageSerializer - } - - -class ApproverSerializer(DocumentSerializer): +class ApproverSerializer(ExampleSerializer): class Meta: - model = Document + model = Example fields = ('id', 'annotation_approver') diff --git a/backend/api/tasks.py b/backend/api/tasks.py index 4a5c8b45..9c1eb486 100644 --- a/backend/api/tasks.py +++ b/backend/api/tasks.py @@ -6,7 +6,7 @@ from django.conf import settings from django.contrib.auth import get_user_model from django.shortcuts import get_object_or_404 -from .models import Document, Label, Project +from .models import Example, Label, Project from .views.download.factory import create_repository, create_writer from .views.download.service import ExportApplicationService from .views.upload.exception import FileParseException @@ -102,7 +102,7 @@ def injest_data(user_id, project_id, filenames, format: str, **kwargs): it = iter(dataset) buffer = Buffer() factory = DataFactory( - data_class=Document, + data_class=Example, label_class=Label, annotation_class=project.get_annotation_class() ) diff --git a/backend/api/tests/api/test_statistics.py b/backend/api/tests/api/test_statistics.py index be5eb555..15c5298e 100644 --- a/backend/api/tests/api/test_statistics.py +++ b/backend/api/tests/api/test_statistics.py @@ -4,7 +4,7 @@ from model_mommy import mommy from rest_framework.reverse import reverse from rest_framework.test import APITestCase -from ...models import DOCUMENT_CLASSIFICATION, Document +from ...models import DOCUMENT_CLASSIFICATION, Example from .utils import (TestUtilsMixin, assign_user_to_role, create_default_roles, remove_all_role_mappings) @@ -32,12 +32,12 @@ class TestStatisticsAPI(APITestCase, TestUtilsMixin): project_type=DOCUMENT_CLASSIFICATION, users=[super_user] ) - doc1 = mommy.make('Document', project=cls.project) - doc2 = mommy.make('Document', project=cls.project) + doc1 = mommy.make('Example', project=cls.project) + doc2 = mommy.make('Example', project=cls.project) mommy.make('Category', example=doc1, user=super_user) mommy.make('Category', example=doc2, user=other_user) cls.url = reverse(viewname='statistics', args=[cls.project.id]) - cls.doc = Document.objects.filter(project=cls.project) + cls.doc = Example.objects.filter(project=cls.project) assign_user_to_role(project_member=other_user, project=cls.project, role_name=settings.ROLE_ANNOTATOR) diff --git a/backend/api/tests/api/utils.py b/backend/api/tests/api/utils.py index 72305d47..2229b3f7 100644 --- a/backend/api/tests/api/utils.py +++ b/backend/api/tests/api/utils.py @@ -89,11 +89,11 @@ def make_label(project): def make_doc(project): - return mommy.make('Document', project=project) + return mommy.make('Example', project=project) def make_image(project): - return mommy.make('Image', project=project) + return mommy.make('Example', project=project) def make_comment(doc, user): diff --git a/backend/api/views/auto_labeling.py b/backend/api/views/auto_labeling.py index 46b5cb7d..d2730671 100644 --- a/backend/api/views/auto_labeling.py +++ b/backend/api/views/auto_labeling.py @@ -16,7 +16,7 @@ from rest_framework.views import APIView from ..exceptions import (AutoLabeliingPermissionDenied, AutoLabelingException, AWSTokenError, SampleDataException, URLConnectionError) -from ..models import AutoLabelingConfig, Document, Project +from ..models import AutoLabelingConfig, Example, Project from ..permissions import IsInProjectOrAdmin, IsProjectAdmin from ..serializers import (AutoLabelingConfigSerializer, get_annotation_serializer) @@ -199,7 +199,7 @@ class AutoLabelingAnnotation(generics.CreateAPIView): def extract(self): project = get_object_or_404(Project, pk=self.kwargs['project_id']) - doc = get_object_or_404(Document, pk=self.kwargs['doc_id']) + doc = get_object_or_404(Example, pk=self.kwargs['doc_id']) config = project.auto_labeling_config.first() if not config: raise AutoLabeliingPermissionDenied() diff --git a/backend/api/views/download/repositories.py b/backend/api/views/download/repositories.py index aa8f09c1..2ddbf823 100644 --- a/backend/api/views/download/repositories.py +++ b/backend/api/views/download/repositories.py @@ -3,7 +3,7 @@ import itertools from collections import defaultdict from typing import Dict, Iterator, List -from ...models import Document, Project +from ...models import Example, Project from .data import Record @@ -66,7 +66,7 @@ class TextRepository(BaseRepository): @property def docs(self): - return Document.objects.filter(project=self.project) + return Example.objects.filter(project=self.project) def list(self, export_approved=False): docs = self.docs @@ -113,7 +113,7 @@ class TextClassificationRepository(TextRepository): @property def docs(self): - return Document.objects.filter(project=self.project).prefetch_related( + return Example.objects.filter(project=self.project).prefetch_related( 'categories__user', 'categories__label' ) @@ -128,7 +128,7 @@ class SequenceLabelingRepository(TextRepository): @property def docs(self): - return Document.objects.filter(project=self.project).prefetch_related( + return Example.objects.filter(project=self.project).prefetch_related( 'spans__user', 'spans__label' ) @@ -144,7 +144,7 @@ class Seq2seqRepository(TextRepository): @property def docs(self): - return Document.objects.filter(project=self.project).prefetch_related( + return Example.objects.filter(project=self.project).prefetch_related( 'texts__user' ) diff --git a/backend/api/views/example.py b/backend/api/views/example.py index 8d200e3e..077b2bfb 100644 --- a/backend/api/views/example.py +++ b/backend/api/views/example.py @@ -8,10 +8,9 @@ from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response from ..filters import DocumentFilter -from ..models import Document, Example, Image, Project +from ..models import Example, Project from ..permissions import IsInProjectReadOnlyOrAdmin -from ..serializers import (DocumentSerializer, ExampleSerializer, - ImageSerializer) +from ..serializers import ExampleSerializer class ExampleList(generics.ListCreateAPIView): @@ -50,27 +49,23 @@ class ExampleList(generics.ListCreateAPIView): class DocumentList(ExampleList): - serializer_class = DocumentSerializer search_fields = ('text',) filter_class = DocumentFilter - model = Document class ImageList(ExampleList): - serializer_class = ImageSerializer search_fields = ('filename',) - model = Image class DocumentDetail(generics.RetrieveUpdateDestroyAPIView): - queryset = Document.objects.all() - serializer_class = DocumentSerializer + queryset = Example.objects.all() + serializer_class = ExampleSerializer lookup_url_kwarg = 'doc_id' permission_classes = [IsAuthenticated & IsInProjectReadOnlyOrAdmin] class ImageDetail(generics.RetrieveUpdateDestroyAPIView): - queryset = Image.objects.all() - serializer_class = ImageSerializer + queryset = Example.objects.all() + serializer_class = ExampleSerializer lookup_url_kwarg = 'image_id' permission_classes = [IsAuthenticated & IsInProjectReadOnlyOrAdmin]