diff --git a/Pipfile.lock b/Pipfile.lock index 3b92eb7d..55e5095f 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -34,11 +34,11 @@ }, "auto-labeling-pipeline": { "hashes": [ - "sha256:29c7771288042f29adfff086c8b0c521ae7fa28b7f83bf62da40a26c6162daa7", - "sha256:9a9443782300304fed53d4dd901d76319b50c5eca2996c7aae3d50d3be56e181" + "sha256:8cb950f9b7ef083468442100fa46ce8c3b5a1e1721d96691d7c07e82aa28de31", + "sha256:ca7b600b7117606c21d77e122b0f0fed2e9c6b9aa6630a76e30750b06ea403cb" ], "index": "pypi", - "version": "==0.1.5" + "version": "==0.1.6" }, "boto3": { "hashes": [ @@ -372,58 +372,58 @@ }, "markupsafe": { "hashes": [ - "sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d", + "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473", + "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161", + "sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235", "sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5", - "sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183", - "sha256:98bae9582248d6cf62321dcb52aaf5d9adf0bad3b40582925ef7c7f0ed85fceb", - "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6", - "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b", - "sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1", - "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e", - "sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f", - "sha256:3b8a6499709d29c2e2399569d96719a1b21dcd94410a586a18526b143ec8470f", - "sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21", - "sha256:2beec1e0de6924ea551859edb9e7679da6e4870d32cb766240ce17e0a0ba2014", + "sha256:13d3144e1e340870b25e7b10b98d779608c02016d5184cfb9927a9f10c689f42", + "sha256:195d7d2c4fbb0ee8139a6cf67194f3973a6b3042d742ebe0a9ed36d8b6f0c07f", "sha256:22c178a091fc6630d0d045bdb5992d2dfe14e3259760e713c490da5323866c39", - "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7", - "sha256:caabedc8323f1e93231b52fc32bdcde6db817623d33e100708d9a68e1f53b26b", - "sha256:feb7b34d6325451ef96bc0e36e1a6c0c1c64bc1fbec4b854f4529e51887b1621", - "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2", - "sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e", - "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c", - "sha256:d9be0ba6c527163cbed5e0857c451fcd092ce83947944d6c14bc95441203f032", "sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff", - "sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f", - "sha256:84dee80c15f1b560d55bcfe6d47b27d070b4681c699c572af2e3c7cc90a3b8e0", - "sha256:195d7d2c4fbb0ee8139a6cf67194f3973a6b3042d742ebe0a9ed36d8b6f0c07f", - "sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b", + "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b", + "sha256:2beec1e0de6924ea551859edb9e7679da6e4870d32cb766240ce17e0a0ba2014", + "sha256:3b8a6499709d29c2e2399569d96719a1b21dcd94410a586a18526b143ec8470f", + "sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1", + "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e", + "sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183", "sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66", - "sha256:6fffc775d90dcc9aed1b89219549b329a9250d918fd0b8fa8d93d154918422e1", "sha256:596510de112c685489095da617b5bcbbac7dd6384aeebeda4df6025d0256a81b", + "sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1", + "sha256:6788b695d50a51edb699cb55e35487e430fa21f1ed838122d722e0ff0ac5ba15", + "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1", + "sha256:6f1e273a344928347c1290119b493a1f0303c52f5a5eae5f16d74f48c15d4a85", + "sha256:6fffc775d90dcc9aed1b89219549b329a9250d918fd0b8fa8d93d154918422e1", + "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e", + "sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b", + "sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905", "sha256:7fed13866cf14bba33e7176717346713881f56d9d2bcebab207f7a036f41b850", + "sha256:84dee80c15f1b560d55bcfe6d47b27d070b4681c699c572af2e3c7cc90a3b8e0", + "sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735", + "sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d", + "sha256:98bae9582248d6cf62321dcb52aaf5d9adf0bad3b40582925ef7c7f0ed85fceb", + "sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e", + "sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d", + "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c", + "sha256:a6a744282b7718a2a62d2ed9d993cad6f5f585605ad352c11de459f4108df0a1", + "sha256:acf08ac40292838b3cbbb06cfe9b2cb9ec78fce8baca31ddb87aaac2e2dc3bc2", + "sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21", + "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2", + "sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5", "sha256:b1dba4527182c95a0db8b6060cc98ac49b9e2f5e64320e2b56e47cb2831978c7", + "sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b", "sha256:b7d644ddb4dbd407d31ffb699f1d140bc35478da613b441c582aeb7c43838dd8", - "sha256:d73a845f227b0bfe8a7455ee623525ee656a9e2e749e4742706d80a6065d5e2c", - "sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5", - "sha256:acf08ac40292838b3cbbb06cfe9b2cb9ec78fce8baca31ddb87aaac2e2dc3bc2", - "sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d", - "sha256:d53bc011414228441014aa71dbec320c66468c1030aae3a6e29778a3382d96e5", - "sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235", - "sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1", - "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1", - "sha256:6788b695d50a51edb699cb55e35487e430fa21f1ed838122d722e0ff0ac5ba15", + "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6", + "sha256:bf5aa3cbcfdf57fa2ee9cd1822c862ef23037f5c832ad09cfea57fa846dec193", + "sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f", + "sha256:caabedc8323f1e93231b52fc32bdcde6db817623d33e100708d9a68e1f53b26b", + "sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f", "sha256:cdb132fc825c38e1aeec2c8aa9338310d29d337bebbd7baa06889d09a60a1fa2", - "sha256:13d3144e1e340870b25e7b10b98d779608c02016d5184cfb9927a9f10c689f42", - "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473", - "sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735", - "sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905", - "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161", - "sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b", + "sha256:d53bc011414228441014aa71dbec320c66468c1030aae3a6e29778a3382d96e5", + "sha256:d73a845f227b0bfe8a7455ee623525ee656a9e2e749e4742706d80a6065d5e2c", + "sha256:d9be0ba6c527163cbed5e0857c451fcd092ce83947944d6c14bc95441203f032", + "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7", "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be", - "sha256:6f1e273a344928347c1290119b493a1f0303c52f5a5eae5f16d74f48c15d4a85", - "sha256:a6a744282b7718a2a62d2ed9d993cad6f5f585605ad352c11de459f4108df0a1", - "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e", - "sha256:bf5aa3cbcfdf57fa2ee9cd1822c862ef23037f5c832ad09cfea57fa846dec193" + "sha256:feb7b34d6325451ef96bc0e36e1a6c0c1c64bc1fbec4b854f4529e51887b1621" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1.1" diff --git a/app/api/migrations/0005_auto_20210201_0603.py b/app/api/migrations/0005_auto_20210201_0603.py new file mode 100644 index 00000000..b78f5de9 --- /dev/null +++ b/app/api/migrations/0005_auto_20210201_0603.py @@ -0,0 +1,33 @@ +# Generated by Django 3.1.5 on 2021-02-01 06:03 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0004_merge_20210114_1117'), + ] + + operations = [ + migrations.AlterField( + model_name='label', + name='suffix_key', + field=models.CharField(blank=True, choices=[('0', '0'), ('1', '1'), ('2', '2'), ('3', '3'), ('4', '4'), ('5', '5'), ('6', '6'), ('7', '7'), ('8', '8'), ('9', '9'), ('a', 'a'), ('b', 'b'), ('c', 'c'), ('d', 'd'), ('e', 'e'), ('f', 'f'), ('g', 'g'), ('h', 'h'), ('i', 'i'), ('j', 'j'), ('k', 'k'), ('l', 'l'), ('m', 'm'), ('n', 'n'), ('o', 'o'), ('p', 'p'), ('q', 'q'), ('r', 'r'), ('s', 's'), ('t', 't'), ('u', 'u'), ('v', 'v'), ('w', 'w'), ('x', 'x'), ('y', 'y'), ('z', 'z')], max_length=1, null=True), + ), + migrations.CreateModel( + name='AutoLabelingConfig', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('model_name', models.CharField(max_length=100)), + ('model_attrs', models.JSONField(default=dict)), + ('template', models.TextField(default='')), + ('label_mapping', models.JSONField(default=dict)), + ('default', models.BooleanField(default=False)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='auto_labeling_config', to='api.project')), + ], + ), + ] diff --git a/app/api/models.py b/app/api/models.py index 119a4697..a29933f4 100644 --- a/app/api/models.py +++ b/app/api/models.py @@ -343,3 +343,17 @@ def delete_linked_project(sender, instance, using, **kwargs): project = Project.objects.get(pk=projectInstance.pk) user.projects.remove(project) user.save() + + +class AutoLabelingConfig(models.Model): + model_name = models.CharField(max_length=100) + model_attrs = models.JSONField(default=dict) + template = models.TextField(default='') + label_mapping = models.JSONField(default=dict) + project = models.ForeignKey(Project, related_name='auto_labeling_config', on_delete=models.CASCADE) + default = models.BooleanField(default=False) + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + + def __str__(self): + return self.model_name diff --git a/app/api/serializers.py b/app/api/serializers.py index 8c3d1950..f652b086 100644 --- a/app/api/serializers.py +++ b/app/api/serializers.py @@ -6,7 +6,7 @@ from rest_polymorphic.serializers import PolymorphicSerializer from rest_framework.exceptions import ValidationError -from .models import Label, Project, Document, RoleMapping, Role, Comment +from .models import Label, Project, Document, RoleMapping, Role, Comment, AutoLabelingConfig from .models import TextClassificationProject, SequenceLabelingProject, Seq2seqProject, Speech2textProject from .models import DocumentAnnotation, SequenceAnnotation, Seq2seqAnnotation, Speech2textAnnotation @@ -241,3 +241,11 @@ class RoleMappingSerializer(serializers.ModelSerializer): class Meta: model = RoleMapping fields = ('id', 'user', 'role', 'username', 'rolename') + + +class AutoLabelingConfigSerializer(serializers.ModelSerializer): + + class Meta: + model = AutoLabelingConfig + fields = ('id', 'model_name', 'model_attrs', 'template', 'label_mapping', 'default') + read_only_fields = ('created_at', 'updated_at') diff --git a/app/api/urls.py b/app/api/urls.py index f0458c0c..edd49967 100644 --- a/app/api/urls.py +++ b/app/api/urls.py @@ -12,6 +12,7 @@ from .views import TextUploadAPI, TextDownloadAPI, CloudUploadAPI from .views import StatisticsAPI from .views import RoleMappingList, RoleMappingDetail, Roles from .views import AutoLabelingTemplateListAPI, AutoLabelingTemplateDetailAPI +from .views import AutoLabelingConfigList, AutoLabelingConfigDetail urlpatterns = [ path('health', Health.as_view(), name='health'), @@ -62,7 +63,17 @@ urlpatterns = [ route='projects//auto-labeling-templates/', view=AutoLabelingTemplateDetailAPI.as_view(), name='auto_labeling_template' - ) + ), + path( + route='projects//auto-labeling-configs', + view=AutoLabelingConfigList.as_view(), + name='auto_labeling_configs' + ), + path( + route='projects//auto-labeling-configs/', + view=AutoLabelingConfigDetail.as_view(), + name='auto_labeling_config' + ), ] # urlpatterns = format_suffix_patterns(urlpatterns, allowed=['json', 'xml']) diff --git a/app/api/views.py b/app/api/views.py index f64b316b..5956ec36 100644 --- a/app/api/views.py +++ b/app/api/views.py @@ -21,10 +21,10 @@ from rest_framework.parsers import MultiPartParser from rest_framework_csv.renderers import CSVRenderer from .filters import DocumentFilter -from .models import Project, Label, Document, RoleMapping, Role, Comment +from .models import Project, Label, Document, RoleMapping, Role, Comment, AutoLabelingConfig from .permissions import IsProjectAdmin, IsAnnotatorAndReadOnly, IsAnnotator, IsAnnotationApproverAndReadOnly, IsOwnAnnotation, IsAnnotationApprover, IsOwnComment from .serializers import ProjectSerializer, LabelSerializer, DocumentSerializer, UserSerializer, ApproverSerializer, CommentSerializer -from .serializers import ProjectPolymorphicSerializer, RoleMappingSerializer, RoleSerializer +from .serializers import ProjectPolymorphicSerializer, RoleMappingSerializer, RoleSerializer, AutoLabelingConfigSerializer from .utils import CSVParser, ExcelParser, JSONParser, PlainTextParser, FastTextParser, CoNLLParser, AudioParser, iterable_to_io from .utils import JSONLRenderer, PlainTextRenderer from .utils import JSONPainter, CSVPainter, FastTextPainter @@ -501,3 +501,24 @@ class AutoLabelingTemplateDetailAPI(APIView): option_name = self.kwargs['option_name'] option = Options.find(option_name=option_name) return Response(option.to_dict(), status=status.HTTP_200_OK) + + +class AutoLabelingConfigList(generics.ListCreateAPIView): + serializer_class = AutoLabelingConfigSerializer + pagination_class = None + permission_classes = [IsAuthenticated & IsProjectAdmin] + + def get_queryset(self): + project = get_object_or_404(Project, pk=self.kwargs['project_id']) + return project.auto_labeling_config + + def perform_create(self, serializer): + project = get_object_or_404(Project, pk=self.kwargs['project_id']) + serializer.save(project=project) + + +class AutoLabelingConfigDetail(generics.RetrieveUpdateDestroyAPIView): + queryset = AutoLabelingConfig.objects.all() + serializer_class = AutoLabelingConfigSerializer + lookup_url_kwarg = 'config_id' + permission_classes = [IsAuthenticated & IsProjectAdmin]