diff --git a/backend/api/admin.py b/backend/api/admin.py index b72ffdb7..941f086b 100644 --- a/backend/api/admin.py +++ b/backend/api/admin.py @@ -1,22 +1,7 @@ from django.contrib import admin -from .models import (CategoryType, Comment, Example, Project, Seq2seqProject, - SequenceLabelingProject, SpanType, Tag, - TextClassificationProject) - - -class LabelAdmin(admin.ModelAdmin): - list_display = ('text', 'project', 'text_color', 'background_color') - ordering = ('project',) - search_fields = ('text',) - - -class CategoryTypeAdmin(LabelAdmin): - pass - - -class SpanTypeAdmin(LabelAdmin): - pass +from .models import (Comment, Example, Project, Seq2seqProject, + SequenceLabelingProject, Tag, TextClassificationProject) class ExampleAdmin(admin.ModelAdmin): @@ -43,8 +28,6 @@ class CommentAdmin(admin.ModelAdmin): search_fields = ('user',) -admin.site.register(CategoryType, CategoryTypeAdmin) -admin.site.register(SpanType, SpanTypeAdmin) admin.site.register(Example, ExampleAdmin) admin.site.register(Project, ProjectAdmin) admin.site.register(TextClassificationProject, ProjectAdmin) diff --git a/backend/api/migrations/0018_alter_label_background_color.py b/backend/api/migrations/0018_alter_label_background_color.py index ca7ec81e..3cf13c2c 100644 --- a/backend/api/migrations/0018_alter_label_background_color.py +++ b/backend/api/migrations/0018_alter_label_background_color.py @@ -1,8 +1,9 @@ # Generated by Django 3.2.8 on 2021-11-17 05:56 -import api.models from django.db import migrations, models +from label_types.models import generate_random_hex_color + class Migration(migrations.Migration): @@ -14,6 +15,6 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='label', name='background_color', - field=models.CharField(default=api.models.generate_random_hex_color, max_length=7), + field=models.CharField(default=generate_random_hex_color, max_length=7), ), ] diff --git a/backend/api/migrations/0020_auto_20211221_1415.py b/backend/api/migrations/0020_auto_20211221_1415.py index b0c1bcc5..5561816b 100644 --- a/backend/api/migrations/0020_auto_20211221_1415.py +++ b/backend/api/migrations/0020_auto_20211221_1415.py @@ -1,9 +1,10 @@ # Generated by Django 3.2.8 on 2021-12-21 14:15 -import api.models from django.db import migrations, models import django.db.models.deletion +from label_types.models import generate_random_hex_color + class Migration(migrations.Migration): @@ -19,7 +20,7 @@ class Migration(migrations.Migration): ('text', models.CharField(db_index=True, max_length=100)), ('prefix_key', models.CharField(blank=True, choices=[('ctrl', 'ctrl'), ('shift', 'shift'), ('ctrl shift', 'ctrl shift')], max_length=10, null=True)), ('suffix_key', models.CharField(blank=True, choices=[('0', '0'), ('1', '1'), ('2', '2'), ('3', '3'), ('4', '4'), ('5', '5'), ('6', '6'), ('7', '7'), ('8', '8'), ('9', '9'), ('a', 'a'), ('b', 'b'), ('c', 'c'), ('d', 'd'), ('e', 'e'), ('f', 'f'), ('g', 'g'), ('h', 'h'), ('i', 'i'), ('j', 'j'), ('k', 'k'), ('l', 'l'), ('m', 'm'), ('n', 'n'), ('o', 'o'), ('p', 'p'), ('q', 'q'), ('r', 'r'), ('s', 's'), ('t', 't'), ('u', 'u'), ('v', 'v'), ('w', 'w'), ('x', 'x'), ('y', 'y'), ('z', 'z')], max_length=1, null=True)), - ('background_color', models.CharField(default=api.models.generate_random_hex_color, max_length=7)), + ('background_color', models.CharField(default=generate_random_hex_color, max_length=7)), ('text_color', models.CharField(default='#ffffff', max_length=7)), ('created_at', models.DateTimeField(auto_now_add=True, db_index=True)), ('updated_at', models.DateTimeField(auto_now=True)), @@ -36,7 +37,7 @@ class Migration(migrations.Migration): ('text', models.CharField(db_index=True, max_length=100)), ('prefix_key', models.CharField(blank=True, choices=[('ctrl', 'ctrl'), ('shift', 'shift'), ('ctrl shift', 'ctrl shift')], max_length=10, null=True)), ('suffix_key', models.CharField(blank=True, choices=[('0', '0'), ('1', '1'), ('2', '2'), ('3', '3'), ('4', '4'), ('5', '5'), ('6', '6'), ('7', '7'), ('8', '8'), ('9', '9'), ('a', 'a'), ('b', 'b'), ('c', 'c'), ('d', 'd'), ('e', 'e'), ('f', 'f'), ('g', 'g'), ('h', 'h'), ('i', 'i'), ('j', 'j'), ('k', 'k'), ('l', 'l'), ('m', 'm'), ('n', 'n'), ('o', 'o'), ('p', 'p'), ('q', 'q'), ('r', 'r'), ('s', 's'), ('t', 't'), ('u', 'u'), ('v', 'v'), ('w', 'w'), ('x', 'x'), ('y', 'y'), ('z', 'z')], max_length=1, null=True)), - ('background_color', models.CharField(default=api.models.generate_random_hex_color, max_length=7)), + ('background_color', models.CharField(default=generate_random_hex_color, max_length=7)), ('text_color', models.CharField(default='#ffffff', max_length=7)), ('created_at', models.DateTimeField(auto_now_add=True, db_index=True)), ('updated_at', models.DateTimeField(auto_now=True)), diff --git a/backend/api/migrations/0032_auto_20220127_0654.py b/backend/api/migrations/0032_auto_20220127_0654.py new file mode 100644 index 00000000..a65fc146 --- /dev/null +++ b/backend/api/migrations/0032_auto_20220127_0654.py @@ -0,0 +1,34 @@ +# Generated by Django 3.2.11 on 2022-01-27 06:54 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0031_auto_20220127_0032'), + ] + + operations = [ + migrations.SeparateDatabaseAndState( + state_operations=[ + migrations.RemoveField( + model_name='categorytype', + name='project', + ), + migrations.AlterUniqueTogether( + name='relationtypes', + unique_together=None, + ), + migrations.RemoveField( + model_name='relationtypes', + name='project', + ), + migrations.RemoveField( + model_name='spantype', + name='project', + ), + ], + database_operations=[] + ) + ] diff --git a/backend/api/migrations/0033_auto_20220127_0654.py b/backend/api/migrations/0033_auto_20220127_0654.py new file mode 100644 index 00000000..e6da429e --- /dev/null +++ b/backend/api/migrations/0033_auto_20220127_0654.py @@ -0,0 +1,41 @@ +# Generated by Django 3.2.11 on 2022-01-27 06:54 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('labels', '0003_auto_20220127_0654'), + ('api', '0032_auto_20220127_0654'), + ] + + operations = [ + migrations.SeparateDatabaseAndState( + state_operations=[ + migrations.DeleteModel( + name='CategoryType', + ), + migrations.DeleteModel( + name='RelationTypes', + ), + migrations.DeleteModel( + name='SpanType', + ), + ], + database_operations=[ + migrations.AlterModelTable( + name='CategoryType', + table='label_types_categorytype' + ), + migrations.AlterModelTable( + name='RelationTypes', + table='label_types_relationtypes' + ), + migrations.AlterModelTable( + name='SpanType', + table='label_types_spanType' + ) + ] + ) + ] diff --git a/backend/api/models.py b/backend/api/models.py index 17958267..6dc967a5 100644 --- a/backend/api/models.py +++ b/backend/api/models.py @@ -1,10 +1,7 @@ import abc -import random -import string import uuid from django.contrib.auth.models import User -from django.core.exceptions import ValidationError from django.db import models from polymorphic.models import PolymorphicModel @@ -151,87 +148,6 @@ class ImageClassificationProject(Project): return True -def generate_random_hex_color(): - return f'#{random.randint(0, 0xFFFFFF):06x}' - - -class Label(models.Model): - text = models.CharField(max_length=100, db_index=True) - prefix_key = models.CharField( - max_length=10, - blank=True, - null=True, - choices=( - ('ctrl', 'ctrl'), - ('shift', 'shift'), - ('ctrl shift', 'ctrl shift') - ) - ) - suffix_key = models.CharField( - max_length=1, - blank=True, - null=True, - choices=tuple( - (c, c) for c in string.digits + string.ascii_lowercase - ) - ) - project = models.ForeignKey( - to=Project, - on_delete=models.CASCADE, - # related_name='labels' - ) - background_color = models.CharField(max_length=7, default=generate_random_hex_color) - text_color = models.CharField(max_length=7, default='#ffffff') - created_at = models.DateTimeField(auto_now_add=True, db_index=True) - updated_at = models.DateTimeField(auto_now=True) - - def __str__(self): - return self.text - - @property - def labels(self): - raise NotImplementedError() - - def clean(self): - # Don't allow shortcut key not to have a suffix key. - if self.prefix_key and not self.suffix_key: - message = 'Shortcut key may not have a suffix key.' - raise ValidationError(message) - - # each shortcut (prefix key + suffix key) can only be assigned to one label - if self.suffix_key or self.prefix_key: - other_labels = self.labels.exclude(id=self.id) - if other_labels.filter(suffix_key=self.suffix_key, prefix_key=self.prefix_key).exists(): - message = 'A label with the shortcut already exists in the project.' - raise ValidationError(message) - - super().clean() - - class Meta: - abstract = True - constraints = [ - models.UniqueConstraint( - fields=['project', 'text'], - name='%(app_label)s_%(class)s_is_unique' - ) - ] - ordering = ['created_at'] - - -class CategoryType(Label): - - @property - def labels(self): - return CategoryType.objects.filter(project=self.project) - - -class SpanType(Label): - - @property - def labels(self): - return SpanType.objects.filter(project=self.project) - - class Example(models.Model): objects = ExampleManager() @@ -318,15 +234,3 @@ class Tag(models.Model): def __str__(self): return self.text - - -class RelationTypes(models.Model): - color = models.TextField() - name = models.TextField() - project = models.ForeignKey(Project, related_name='relation_types', on_delete=models.CASCADE) - - def __str__(self): - return self.name - - class Meta: - unique_together = ('color', 'name') diff --git a/backend/api/serializers.py b/backend/api/serializers.py index 0c632ad2..7bfc890e 100644 --- a/backend/api/serializers.py +++ b/backend/api/serializers.py @@ -1,87 +1,11 @@ from rest_framework import serializers -from rest_framework.exceptions import ValidationError from rest_polymorphic.serializers import PolymorphicSerializer -from .models import (CategoryType, Comment, Example, ExampleState, +from .models import (Comment, Example, ExampleState, ImageClassificationProject, - IntentDetectionAndSlotFillingProject, Label, Project, - RelationTypes, Seq2seqProject, SequenceLabelingProject, - SpanType, Speech2textProject, Tag, - TextClassificationProject) - - -class LabelSerializer(serializers.ModelSerializer): - - def validate(self, attrs): - prefix_key = attrs.get('prefix_key') - suffix_key = attrs.get('suffix_key') - - # In the case of user don't set any shortcut key. - if prefix_key is None and suffix_key is None: - return super().validate(attrs) - - # Don't allow shortcut key not to have a suffix key. - if prefix_key and not suffix_key: - raise ValidationError('Shortcut key may not have a suffix key.') - - # Don't allow to save same shortcut key when prefix_key is null. - try: - context = self.context['request'].parser_context - project_id = context['kwargs']['project_id'] - label_id = context['kwargs'].get('label_id') - except (AttributeError, KeyError): - pass # unit tests don't always have the correct context set up - else: - conflicting_labels = self.Meta.model.objects.filter( - suffix_key=suffix_key, - prefix_key=prefix_key, - project=project_id, - ) - - if label_id is not None: - conflicting_labels = conflicting_labels.exclude(id=label_id) - - if conflicting_labels.exists(): - raise ValidationError('Duplicate shortcut key.') - - return super().validate(attrs) - - class Meta: - model = Label - fields = ( - 'id', - 'text', - 'prefix_key', - 'suffix_key', - 'background_color', - 'text_color', - ) - - -class CategoryTypeSerializer(LabelSerializer): - class Meta: - model = CategoryType - fields = ( - 'id', - 'text', - 'prefix_key', - 'suffix_key', - 'background_color', - 'text_color', - ) - - -class SpanTypeSerializer(LabelSerializer): - class Meta: - model = SpanType - fields = ( - 'id', - 'text', - 'prefix_key', - 'suffix_key', - 'background_color', - 'text_color', - ) + IntentDetectionAndSlotFillingProject, Project, + Seq2seqProject, SequenceLabelingProject, + Speech2textProject, Tag, TextClassificationProject) class CommentSerializer(serializers.ModelSerializer): @@ -139,13 +63,6 @@ class ExampleStateSerializer(serializers.ModelSerializer): read_only_fields = ('id', 'example', 'confirmed_by') -class ApproverSerializer(ExampleSerializer): - - class Meta: - model = Example - fields = ('id', 'annotation_approver') - - class ProjectSerializer(serializers.ModelSerializer): tags = TagSerializer(many=True, required=False) @@ -224,13 +141,3 @@ class ProjectPolymorphicSerializer(PolymorphicSerializer): cls.Meta.model: cls for cls in ProjectSerializer.__subclasses__() } } - - -class RelationTypesSerializer(serializers.ModelSerializer): - - def validate(self, attrs): - return super().validate(attrs) - - class Meta: - model = RelationTypes - fields = ('id', 'color', 'name') diff --git a/backend/api/tests/test_models.py b/backend/api/tests/test_models.py index e9a05e17..8d1bf38f 100644 --- a/backend/api/tests/test_models.py +++ b/backend/api/tests/test_models.py @@ -1,73 +1,11 @@ -from django.core.exceptions import ValidationError -from django.db.utils import IntegrityError from django.test import TestCase from model_mommy import mommy -from api.models import (IMAGE_CLASSIFICATION, SEQUENCE_LABELING, CategoryType, - ExampleState, generate_random_hex_color) +from api.models import IMAGE_CLASSIFICATION, SEQUENCE_LABELING, ExampleState from .api.utils import prepare_project -class TestLabel(TestCase): - - def test_deny_creating_same_text(self): - label = mommy.make('CategoryType') - with self.assertRaises(IntegrityError): - mommy.make('CategoryType', project=label.project, text=label.text) - - def test_keys_uniqueness(self): - label = mommy.make('CategoryType', prefix_key='ctrl', suffix_key='a') - with self.assertRaises(ValidationError): - CategoryType(project=label.project, - text='example', - prefix_key=label.prefix_key, - suffix_key=label.suffix_key).full_clean() - - def test_suffix_key_uniqueness(self): - label = mommy.make('CategoryType', prefix_key=None, suffix_key='a') - with self.assertRaises(ValidationError): - CategoryType(project=label.project, - text='example', - prefix_key=label.prefix_key, - suffix_key=label.suffix_key).full_clean() - - def test_cannot_add_label_only_prefix_key(self): - project = mommy.make('Project') - label = CategoryType(project=project, - text='example', - prefix_key='ctrl') - with self.assertRaises(ValidationError): - label.clean() - - def test_can_add_label_only_suffix_key(self): - project = mommy.make('Project') - label = CategoryType(project=project, text='example', suffix_key='a') - try: - label.full_clean() - except ValidationError: - self.fail(msg=ValidationError) - - def test_can_add_label_suffix_key_with_prefix_key(self): - project = mommy.make('Project') - label = CategoryType(project=project, - text='example', - prefix_key='ctrl', - suffix_key='a') - try: - label.full_clean() - except ValidationError: - self.fail(msg=ValidationError) - - -class TestGeneratedColor(TestCase): - - def test_length(self): - for i in range(100): - color = generate_random_hex_color() - self.assertEqual(len(color), 7) - - class TestExampleState(TestCase): def setUp(self): diff --git a/backend/api/urls.py b/backend/api/urls.py index ae2dfb98..3115be8b 100644 --- a/backend/api/urls.py +++ b/backend/api/urls.py @@ -1,39 +1,8 @@ from django.urls import include, path -from .views import (comment, example, example_state, health, label, project, - tag, task) +from .views import comment, example, example_state, health, project, tag, task urlpatterns_project = [ - path( - route='category-types', - view=label.CategoryTypeList.as_view(), - name='category_types' - ), - path( - route='category-types/', - view=label.CategoryTypeDetail.as_view(), - name='category_type' - ), - path( - route='span-types', - view=label.SpanTypeList.as_view(), - name='span_types' - ), - path( - route='span-types/', - view=label.SpanTypeDetail.as_view(), - name='span_type' - ), - path( - route='category-type-upload', - view=label.CategoryTypeUploadAPI.as_view(), - name='category_type_upload' - ), - path( - route='span-type-upload', - view=label.SpanTypeUploadAPI.as_view(), - name='span_type_upload' - ), path( route='examples', view=example.ExampleList.as_view(), @@ -44,21 +13,6 @@ urlpatterns_project = [ view=example.ExampleDetail.as_view(), name='example_detail' ), - path( - route='relation_types', - view=label.RelationTypeList.as_view(), - name='relation_types_list' - ), - path( - route='relation_type-upload', - view=label.RelationTypeUploadAPI.as_view(), - name='relation_type-upload' - ), - path( - route='relation_types/', - view=label.RelationTypeDetail.as_view(), - name='relation_type_detail' - ), path( route='tags', view=tag.TagList.as_view(), diff --git a/backend/app/settings.py b/backend/app/settings.py index ff2e93dd..adf1fb59 100644 --- a/backend/app/settings.py +++ b/backend/app/settings.py @@ -60,6 +60,7 @@ INSTALLED_APPS = [ 'data_export.apps.DataExportConfig', 'auto_labeling.apps.AutoLabelingConfig', 'labels.apps.LabelsConfig', + 'label_types.apps.LabelTypesConfig', 'rest_framework', 'rest_framework.authtoken', 'django_filters', diff --git a/backend/app/urls.py b/backend/app/urls.py index e2748636..1c7964d1 100644 --- a/backend/app/urls.py +++ b/backend/app/urls.py @@ -49,6 +49,7 @@ urlpatterns += [ path('v1/projects//metrics/', include('metrics.urls')), path('v1/projects//', include('auto_labeling.urls')), path('v1/projects//', include('labels.urls')), + path('v1/projects//', include('label_types.urls')), path('swagger/', schema_view.with_ui('swagger', cache_timeout=0), name='schema-swagger-ui'), re_path('', TemplateView.as_view(template_name='index.html')), ] diff --git a/backend/auto_labeling/pipeline/labels.py b/backend/auto_labeling/pipeline/labels.py index fabcc91c..98c19b9e 100644 --- a/backend/auto_labeling/pipeline/labels.py +++ b/backend/auto_labeling/pipeline/labels.py @@ -4,7 +4,8 @@ from typing import List from auto_labeling_pipeline.labels import Labels from django.contrib.auth.models import User -from api.models import Project, Example, CategoryType, SpanType +from api.models import Project, Example +from label_types.models import CategoryType, SpanType from labels.models import Label, Category, Span, TextLabel diff --git a/backend/data_import/pipeline/labels.py b/backend/data_import/pipeline/labels.py index 31f57f50..443cd523 100644 --- a/backend/data_import/pipeline/labels.py +++ b/backend/data_import/pipeline/labels.py @@ -3,9 +3,8 @@ from typing import Any, Dict, Optional, Union from pydantic import BaseModel, validator -from api.models import CategoryType -from api.models import Label as LabelModel -from api.models import Project, SpanType +from api.models import Project +from label_types.models import LabelType, CategoryType, SpanType from labels.models import Category, Span, TextLabel as TL @@ -25,7 +24,7 @@ class Label(BaseModel, abc.ABC): raise NotImplementedError() @abc.abstractmethod - def create(self, project: Project) -> Optional[LabelModel]: + def create(self, project: Project) -> Optional[LabelType]: raise NotImplementedError() @abc.abstractmethod @@ -62,10 +61,10 @@ class CategoryLabel(Label): else: raise TypeError(f'{obj} is not str.') - def create(self, project: Project) -> Optional[LabelModel]: + def create(self, project: Project) -> Optional[LabelType]: return CategoryType(text=self.label, project=project) - def create_annotation(self, user, example, mapping: Dict[str, LabelModel]): + def create_annotation(self, user, example, mapping: Dict[str, LabelType]): return Category( user=user, example=example, @@ -96,10 +95,10 @@ class SpanLabel(Label): else: raise TypeError(f'{obj} is invalid type.') - def create(self, project: Project) -> Optional[LabelModel]: + def create(self, project: Project) -> Optional[LabelType]: return SpanType(text=self.label, project=project) - def create_annotation(self, user, example, mapping: Dict[str, LabelModel]): + def create_annotation(self, user, example, mapping: Dict[str, LabelType]): return Span( user=user, example=example, @@ -126,7 +125,7 @@ class TextLabel(Label): else: raise TypeError(f'{obj} is not str or empty.') - def create(self, project: Project) -> Optional[LabelModel]: + def create(self, project: Project) -> Optional[LabelType]: return None def create_annotation(self, user, example, mapping): diff --git a/backend/data_import/pipeline/writers.py b/backend/data_import/pipeline/writers.py index a79bd442..e8636b79 100644 --- a/backend/data_import/pipeline/writers.py +++ b/backend/data_import/pipeline/writers.py @@ -5,7 +5,8 @@ from typing import Any, Dict, List from django.conf import settings -from api.models import CategoryType, Example, Project, SpanType +from api.models import Example, Project +from label_types.models import CategoryType, SpanType from .exceptions import FileParseException from .readers import BaseReader diff --git a/backend/data_import/tests/test_tasks.py b/backend/data_import/tests/test_tasks.py index 3f7f7a62..fed7da64 100644 --- a/backend/data_import/tests/test_tasks.py +++ b/backend/data_import/tests/test_tasks.py @@ -5,7 +5,8 @@ from django.test import TestCase from data_import.celery_tasks import import_dataset from api.models import (DOCUMENT_CLASSIFICATION, INTENT_DETECTION_AND_SLOT_FILLING, SEQ2SEQ, - SEQUENCE_LABELING, CategoryType, Example, SpanType) + SEQUENCE_LABELING, Example) +from label_types.models import CategoryType, SpanType from labels.models import Category, Span from api.tests.api.utils import prepare_project diff --git a/backend/label_types/__init__.py b/backend/label_types/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/label_types/admin.py b/backend/label_types/admin.py new file mode 100644 index 00000000..048ff28b --- /dev/null +++ b/backend/label_types/admin.py @@ -0,0 +1,21 @@ +from django.contrib import admin + +from .models import CategoryType, SpanType + + +class LabelAdmin(admin.ModelAdmin): + list_display = ('text', 'project', 'text_color', 'background_color') + ordering = ('project',) + search_fields = ('text',) + + +class CategoryTypeAdmin(LabelAdmin): + pass + + +class SpanTypeAdmin(LabelAdmin): + pass + + +admin.site.register(CategoryType, CategoryTypeAdmin) +admin.site.register(SpanType, SpanTypeAdmin) diff --git a/backend/label_types/apps.py b/backend/label_types/apps.py new file mode 100644 index 00000000..c7e12088 --- /dev/null +++ b/backend/label_types/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class LabelTypesConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'label_types' diff --git a/backend/api/exceptions.py b/backend/label_types/exceptions.py similarity index 100% rename from backend/api/exceptions.py rename to backend/label_types/exceptions.py diff --git a/backend/label_types/migrations/0001_initial.py b/backend/label_types/migrations/0001_initial.py new file mode 100644 index 00000000..198fd680 --- /dev/null +++ b/backend/label_types/migrations/0001_initial.py @@ -0,0 +1,112 @@ +# Generated by Django 3.2.11 on 2022-01-27 06:54 + +from django.db import migrations, models +import django.db.models.deletion +import label_types.models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('api', '0032_auto_20220127_0654'), + ] + + operations = [ + migrations.SeparateDatabaseAndState( + state_operations=[ + migrations.CreateModel( + name='SpanType', + fields=[ + ('id', + models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('text', models.CharField(db_index=True, max_length=100)), + ('prefix_key', models.CharField(blank=True, choices=[('ctrl', 'ctrl'), ('shift', 'shift'), + ('ctrl shift', 'ctrl shift')], + max_length=10, null=True)), + ('suffix_key', models.CharField(blank=True, + choices=[('0', '0'), ('1', '1'), ('2', '2'), ('3', '3'), + ('4', '4'), ('5', '5'), ('6', '6'), ('7', '7'), + ('8', '8'), ('9', '9'), ('a', 'a'), ('b', 'b'), + ('c', 'c'), ('d', 'd'), ('e', 'e'), ('f', 'f'), + ('g', 'g'), ('h', 'h'), ('i', 'i'), ('j', 'j'), + ('k', 'k'), ('l', 'l'), ('m', 'm'), ('n', 'n'), + ('o', 'o'), ('p', 'p'), ('q', 'q'), ('r', 'r'), + ('s', 's'), ('t', 't'), ('u', 'u'), ('v', 'v'), + ('w', 'w'), ('x', 'x'), ('y', 'y'), ('z', 'z')], + max_length=1, null=True)), + ('background_color', + models.CharField(default=label_types.models.generate_random_hex_color, max_length=7)), + ('text_color', models.CharField(default='#ffffff', max_length=7)), + ('created_at', models.DateTimeField(auto_now_add=True, db_index=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='api.project')), + ], + options={ + 'ordering': ['created_at'], + 'abstract': False, + }, + ), + migrations.CreateModel( + name='RelationTypes', + fields=[ + ('id', + models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('color', models.TextField()), + ('name', models.TextField()), + ('project', + models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='relation_types', + to='api.project')), + ], + ), + migrations.CreateModel( + name='CategoryType', + fields=[ + ('id', + models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('text', models.CharField(db_index=True, max_length=100)), + ('prefix_key', models.CharField(blank=True, choices=[('ctrl', 'ctrl'), ('shift', 'shift'), + ('ctrl shift', 'ctrl shift')], + max_length=10, null=True)), + ('suffix_key', models.CharField(blank=True, + choices=[('0', '0'), ('1', '1'), ('2', '2'), ('3', '3'), + ('4', '4'), ('5', '5'), ('6', '6'), ('7', '7'), + ('8', '8'), ('9', '9'), ('a', 'a'), ('b', 'b'), + ('c', 'c'), ('d', 'd'), ('e', 'e'), ('f', 'f'), + ('g', 'g'), ('h', 'h'), ('i', 'i'), ('j', 'j'), + ('k', 'k'), ('l', 'l'), ('m', 'm'), ('n', 'n'), + ('o', 'o'), ('p', 'p'), ('q', 'q'), ('r', 'r'), + ('s', 's'), ('t', 't'), ('u', 'u'), ('v', 'v'), + ('w', 'w'), ('x', 'x'), ('y', 'y'), ('z', 'z')], + max_length=1, null=True)), + ('background_color', + models.CharField(default=label_types.models.generate_random_hex_color, max_length=7)), + ('text_color', models.CharField(default='#ffffff', max_length=7)), + ('created_at', models.DateTimeField(auto_now_add=True, db_index=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='api.project')), + ], + options={ + 'ordering': ['created_at'], + 'abstract': False, + }, + ), + migrations.AddConstraint( + model_name='spantype', + constraint=models.UniqueConstraint(fields=('project', 'text'), + name='label_types_spantype_is_unique'), + ), + migrations.AlterUniqueTogether( + name='relationtypes', + unique_together={('color', 'name')}, + ), + migrations.AddConstraint( + model_name='categorytype', + constraint=models.UniqueConstraint(fields=('project', 'text'), + name='label_types_categorytype_is_unique'), + ), + ], + database_operations=[] + ) + ] diff --git a/backend/label_types/migrations/0002_rename_relationtypes_relationtype.py b/backend/label_types/migrations/0002_rename_relationtypes_relationtype.py new file mode 100644 index 00000000..561f6d7e --- /dev/null +++ b/backend/label_types/migrations/0002_rename_relationtypes_relationtype.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.11 on 2022-01-27 07:27 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('labels', '0003_auto_20220127_0654'), + ('api', '0033_auto_20220127_0654'), + ('label_types', '0001_initial'), + ] + + operations = [ + migrations.RenameModel( + old_name='RelationTypes', + new_name='RelationType', + ), + ] diff --git a/backend/label_types/migrations/__init__.py b/backend/label_types/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/label_types/models.py b/backend/label_types/models.py new file mode 100644 index 00000000..b340f34b --- /dev/null +++ b/backend/label_types/models.py @@ -0,0 +1,100 @@ +import random +import string + +from django.core.exceptions import ValidationError +from django.db import models + +from api.models import Project + + +def generate_random_hex_color(): + return f'#{random.randint(0, 0xFFFFFF):06x}' + + +class LabelType(models.Model): + text = models.CharField(max_length=100, db_index=True) + prefix_key = models.CharField( + max_length=10, + blank=True, + null=True, + choices=( + ('ctrl', 'ctrl'), + ('shift', 'shift'), + ('ctrl shift', 'ctrl shift') + ) + ) + suffix_key = models.CharField( + max_length=1, + blank=True, + null=True, + choices=tuple( + (c, c) for c in string.digits + string.ascii_lowercase + ) + ) + project = models.ForeignKey( + to=Project, + on_delete=models.CASCADE, + # related_name='labels' + ) + background_color = models.CharField(max_length=7, default=generate_random_hex_color) + text_color = models.CharField(max_length=7, default='#ffffff') + created_at = models.DateTimeField(auto_now_add=True, db_index=True) + updated_at = models.DateTimeField(auto_now=True) + + def __str__(self): + return self.text + + @property + def labels(self): + raise NotImplementedError() + + def clean(self): + # Don't allow shortcut key not to have a suffix key. + if self.prefix_key and not self.suffix_key: + message = 'Shortcut key may not have a suffix key.' + raise ValidationError(message) + + # each shortcut (prefix key + suffix key) can only be assigned to one label + if self.suffix_key or self.prefix_key: + other_labels = self.labels.exclude(id=self.id) + if other_labels.filter(suffix_key=self.suffix_key, prefix_key=self.prefix_key).exists(): + message = 'A label with the shortcut already exists in the project.' + raise ValidationError(message) + + super().clean() + + class Meta: + abstract = True + constraints = [ + models.UniqueConstraint( + fields=['project', 'text'], + name='%(app_label)s_%(class)s_is_unique' + ) + ] + ordering = ['created_at'] + + +class CategoryType(LabelType): + + @property + def labels(self): + return CategoryType.objects.filter(project=self.project) + + +class SpanType(LabelType): + + @property + def labels(self): + return SpanType.objects.filter(project=self.project) + + +class RelationType(models.Model): + color = models.TextField() + name = models.TextField() + project = models.ForeignKey(Project, related_name='relation_types', on_delete=models.CASCADE) + + def __str__(self): + return self.name + + class Meta: + unique_together = ('color', 'name') diff --git a/backend/label_types/serializers.py b/backend/label_types/serializers.py new file mode 100644 index 00000000..5e1368a4 --- /dev/null +++ b/backend/label_types/serializers.py @@ -0,0 +1,88 @@ +from rest_framework import serializers +from rest_framework.exceptions import ValidationError + +from .models import LabelType, CategoryType, SpanType, RelationType + + +class LabelSerializer(serializers.ModelSerializer): + + def validate(self, attrs): + prefix_key = attrs.get('prefix_key') + suffix_key = attrs.get('suffix_key') + + # In the case of user don't set any shortcut key. + if prefix_key is None and suffix_key is None: + return super().validate(attrs) + + # Don't allow shortcut key not to have a suffix key. + if prefix_key and not suffix_key: + raise ValidationError('Shortcut key may not have a suffix key.') + + # Don't allow to save same shortcut key when prefix_key is null. + try: + context = self.context['request'].parser_context + project_id = context['kwargs']['project_id'] + label_id = context['kwargs'].get('label_id') + except (AttributeError, KeyError): + pass # unit tests don't always have the correct context set up + else: + conflicting_labels = self.Meta.model.objects.filter( + suffix_key=suffix_key, + prefix_key=prefix_key, + project=project_id, + ) + + if label_id is not None: + conflicting_labels = conflicting_labels.exclude(id=label_id) + + if conflicting_labels.exists(): + raise ValidationError('Duplicate shortcut key.') + + return super().validate(attrs) + + class Meta: + model = LabelType + fields = ( + 'id', + 'text', + 'prefix_key', + 'suffix_key', + 'background_color', + 'text_color', + ) + + +class CategoryTypeSerializer(LabelSerializer): + class Meta: + model = CategoryType + fields = ( + 'id', + 'text', + 'prefix_key', + 'suffix_key', + 'background_color', + 'text_color', + ) + + +class SpanTypeSerializer(LabelSerializer): + class Meta: + model = SpanType + fields = ( + 'id', + 'text', + 'prefix_key', + 'suffix_key', + 'background_color', + 'text_color', + ) + + +class RelationTypesSerializer(serializers.ModelSerializer): + + def validate(self, attrs): + return super().validate(attrs) + + class Meta: + model = RelationType + fields = ('id', 'color', 'name') diff --git a/backend/label_types/tests/__init__.py b/backend/label_types/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/label_types/tests/test_models.py b/backend/label_types/tests/test_models.py new file mode 100644 index 00000000..61cdded8 --- /dev/null +++ b/backend/label_types/tests/test_models.py @@ -0,0 +1,65 @@ +from django.core.exceptions import ValidationError +from django.db import IntegrityError +from django.test import TestCase +from model_mommy import mommy + +from label_types.models import CategoryType, generate_random_hex_color + + +class TestLabel(TestCase): + + def test_deny_creating_same_text(self): + label = mommy.make('CategoryType') + with self.assertRaises(IntegrityError): + mommy.make('CategoryType', project=label.project, text=label.text) + + def test_keys_uniqueness(self): + label = mommy.make('CategoryType', prefix_key='ctrl', suffix_key='a') + with self.assertRaises(ValidationError): + CategoryType(project=label.project, + text='example', + prefix_key=label.prefix_key, + suffix_key=label.suffix_key).full_clean() + + def test_suffix_key_uniqueness(self): + label = mommy.make('CategoryType', prefix_key=None, suffix_key='a') + with self.assertRaises(ValidationError): + CategoryType(project=label.project, + text='example', + prefix_key=label.prefix_key, + suffix_key=label.suffix_key).full_clean() + + def test_cannot_add_label_only_prefix_key(self): + project = mommy.make('Project') + label = CategoryType(project=project, + text='example', + prefix_key='ctrl') + with self.assertRaises(ValidationError): + label.clean() + + def test_can_add_label_only_suffix_key(self): + project = mommy.make('Project') + label = CategoryType(project=project, text='example', suffix_key='a') + try: + label.full_clean() + except ValidationError: + self.fail(msg=ValidationError) + + def test_can_add_label_suffix_key_with_prefix_key(self): + project = mommy.make('Project') + label = CategoryType(project=project, + text='example', + prefix_key='ctrl', + suffix_key='a') + try: + label.full_clean() + except ValidationError: + self.fail(msg=ValidationError) + + +class TestGeneratedColor(TestCase): + + def test_length(self): + for i in range(100): + color = generate_random_hex_color() + self.assertEqual(len(color), 7) diff --git a/backend/api/tests/api/test_label.py b/backend/label_types/tests/test_views.py similarity index 98% rename from backend/api/tests/api/test_label.py rename to backend/label_types/tests/test_views.py index 4da5a3e1..77513f03 100644 --- a/backend/api/tests/api/test_label.py +++ b/backend/label_types/tests/test_views.py @@ -6,9 +6,7 @@ from rest_framework.reverse import reverse from rest_framework.test import APITestCase from api.models import DOCUMENT_CLASSIFICATION - -from .utils import (DATA_DIR, CRUDMixin, make_label, make_project, make_user, - prepare_project) +from api.tests.api.utils import (DATA_DIR, CRUDMixin, make_label, make_project, make_user, prepare_project) class TestLabelList(CRUDMixin): diff --git a/backend/label_types/urls.py b/backend/label_types/urls.py new file mode 100644 index 00000000..dbed3d88 --- /dev/null +++ b/backend/label_types/urls.py @@ -0,0 +1,54 @@ +from django.urls import path + +from .views import CategoryTypeList, CategoryTypeDetail, CategoryTypeUploadAPI +from .views import SpanTypeList, SpanTypeDetail, SpanTypeUploadAPI +from .views import RelationTypeList, RelationTypeDetail, RelationTypeUploadAPI + + +urlpatterns = [ + path( + route='category-types', + view=CategoryTypeList.as_view(), + name='category_types' + ), + path( + route='category-types/', + view=CategoryTypeDetail.as_view(), + name='category_type' + ), + path( + route='span-types', + view=SpanTypeList.as_view(), + name='span_types' + ), + path( + route='span-types/', + view=SpanTypeDetail.as_view(), + name='span_type' + ), + path( + route='category-type-upload', + view=CategoryTypeUploadAPI.as_view(), + name='category_type_upload' + ), + path( + route='span-type-upload', + view=SpanTypeUploadAPI.as_view(), + name='span_type_upload' + ), + path( + route='relation_types', + view=RelationTypeList.as_view(), + name='relation_types_list' + ), + path( + route='relation_type-upload', + view=RelationTypeUploadAPI.as_view(), + name='relation_type-upload' + ), + path( + route='relation_types/', + view=RelationTypeDetail.as_view(), + name='relation_type_detail' + ), +] diff --git a/backend/api/views/label.py b/backend/label_types/views.py similarity index 91% rename from backend/api/views/label.py rename to backend/label_types/views.py index fdf131d8..8297b3c6 100644 --- a/backend/api/views/label.py +++ b/backend/label_types/views.py @@ -11,11 +11,10 @@ from rest_framework.response import Response from rest_framework.views import APIView from members.permissions import IsInProjectReadOnlyOrAdmin, IsProjectAdmin - -from ..exceptions import LabelValidationError -from ..models import CategoryType, Label, RelationTypes, SpanType -from ..serializers import (CategoryTypeSerializer, LabelSerializer, - RelationTypesSerializer, SpanTypeSerializer) +from .models import LabelType, CategoryType, SpanType, RelationType +from .exceptions import LabelValidationError +from .serializers import (CategoryTypeSerializer, LabelSerializer, + RelationTypesSerializer, SpanTypeSerializer) def camel_to_snake(name): @@ -28,7 +27,7 @@ def camel_to_snake_dict(d): class LabelList(generics.ListCreateAPIView): - model = Label + model = LabelType filter_backends = [DjangoFilterBackend] serializer_class = LabelSerializer pagination_class = None @@ -71,12 +70,12 @@ class SpanTypeDetail(generics.RetrieveUpdateDestroyAPIView): class RelationTypeList(LabelList): - model = RelationTypes + model = RelationType serializer_class = RelationTypesSerializer class RelationTypeDetail(generics.RetrieveUpdateDestroyAPIView): - queryset = RelationTypes.objects.all() + queryset = RelationType.objects.all() serializer_class = RelationTypesSerializer lookup_url_kwarg = 'relation_type_id' permission_classes = [IsAuthenticated & IsInProjectReadOnlyOrAdmin] diff --git a/backend/labels/migrations/0003_auto_20220127_0654.py b/backend/labels/migrations/0003_auto_20220127_0654.py new file mode 100644 index 00000000..d759b0ea --- /dev/null +++ b/backend/labels/migrations/0003_auto_20220127_0654.py @@ -0,0 +1,35 @@ +# Generated by Django 3.2.11 on 2022-01-27 06:54 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('label_types', '0001_initial'), + ('labels', '0002_rename_annotationrelations_relation'), + ] + + operations = [ + migrations.SeparateDatabaseAndState( + state_operations=[ + migrations.AlterField( + model_name='category', + name='label', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='label_types.categorytype'), + ), + migrations.AlterField( + model_name='relation', + name='type', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, + related_name='annotation_relations', to='label_types.relationtypes'), + ), + migrations.AlterField( + model_name='span', + name='label', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='label_types.spantype'), + ), + ] + ) + ] diff --git a/backend/labels/models.py b/backend/labels/models.py index 48b40b5b..5b8690b5 100644 --- a/backend/labels/models.py +++ b/backend/labels/models.py @@ -3,7 +3,8 @@ from django.core.exceptions import ValidationError from django.db import models from .managers import LabelManager, CategoryManager, SpanManager, TextLabelManager -from api.models import Example, CategoryType, SpanType, RelationTypes, Project +from api.models import Example, Project +from label_types.models import CategoryType, SpanType, RelationType class Label(models.Model): @@ -107,7 +108,7 @@ class TextLabel(Label): class Relation(models.Model): annotation_id_1 = models.IntegerField() annotation_id_2 = models.IntegerField() - type = models.ForeignKey(RelationTypes, related_name='annotation_relations', on_delete=models.CASCADE) + type = models.ForeignKey(RelationType, related_name='annotation_relations', on_delete=models.CASCADE) timestamp = models.DateTimeField() user = models.ForeignKey(User, related_name='annotation_relations', on_delete=models.CASCADE) project = models.ForeignKey(Project, related_name='annotation_relations', on_delete=models.CASCADE) diff --git a/backend/labels/serializers.py b/backend/labels/serializers.py index e87c66b8..0652d267 100644 --- a/backend/labels/serializers.py +++ b/backend/labels/serializers.py @@ -1,6 +1,7 @@ from rest_framework import serializers -from api.models import CategoryType, Example, SpanType +from api.models import Example +from label_types.models import CategoryType, SpanType from .models import Category, Span, TextLabel, Relation diff --git a/backend/labels/tests/test_span.py b/backend/labels/tests/test_span.py index 79e32f20..71e25e74 100644 --- a/backend/labels/tests/test_span.py +++ b/backend/labels/tests/test_span.py @@ -5,7 +5,8 @@ from django.db import IntegrityError from django.test import TestCase from model_mommy import mommy -from api.models import SEQUENCE_LABELING, SpanType +from api.models import SEQUENCE_LABELING +from label_types.models import SpanType from labels.models import Span from api.tests.api.utils import prepare_project diff --git a/backend/metrics/views.py b/backend/metrics/views.py index 43b795fc..ca2887f6 100644 --- a/backend/metrics/views.py +++ b/backend/metrics/views.py @@ -5,7 +5,8 @@ from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response from rest_framework.views import APIView -from api.models import Example, ExampleState, CategoryType, SpanType, Label as LabelType +from api.models import Example, ExampleState +from label_types.models import LabelType, CategoryType, SpanType from labels.models import Label, Category, Span from members.models import Member from members.permissions import IsInProjectReadOnlyOrAdmin