From 46c5a0d967c7b97bd7ffd2952aed079c6d29a34c Mon Sep 17 00:00:00 2001
From: Hironsan <light.tree.1.13@gmail.com>
Date: Mon, 4 Jul 2022 12:47:34 +0900
Subject: [PATCH] Support segmentation

---
 frontend/domain/models/tasks/segmentation.ts  |  31 ++
 .../pages/projects/_id/segmentation/index.vue | 321 ++++++++++++++++++
 frontend/plugins/services.ts                  |   8 +-
 .../segmentation/apiSegmentationRepository.ts |  40 +++
 .../segmentationApplicationService.ts         |  44 +++
 .../tasks/segmentation/segmentationData.ts    |  15 +
 6 files changed, 458 insertions(+), 1 deletion(-)
 create mode 100644 frontend/domain/models/tasks/segmentation.ts
 create mode 100644 frontend/pages/projects/_id/segmentation/index.vue
 create mode 100644 frontend/repositories/tasks/segmentation/apiSegmentationRepository.ts
 create mode 100644 frontend/services/application/tasks/segmentation/segmentationApplicationService.ts
 create mode 100644 frontend/services/application/tasks/segmentation/segmentationData.ts

diff --git a/frontend/domain/models/tasks/segmentation.ts b/frontend/domain/models/tasks/segmentation.ts
new file mode 100644
index 00000000..60796464
--- /dev/null
+++ b/frontend/domain/models/tasks/segmentation.ts
@@ -0,0 +1,31 @@
+export class SegmentationItem {
+  constructor(
+    public id: number,
+    public uuid: string,
+    public label: number,
+    public points: number[]
+  ) {}
+
+  static valueOf({
+    id,
+    uuid,
+    label,
+    points
+  }: {
+    id: number
+    uuid: string
+    label: number
+    points: number[]
+  }): SegmentationItem {
+    return new SegmentationItem(id, uuid, label, points)
+  }
+
+  toObject(): Object {
+    return {
+      id: this.id,
+      uuid: this.uuid,
+      label: this.label,
+      points: this.points
+    }
+  }
+}
diff --git a/frontend/pages/projects/_id/segmentation/index.vue b/frontend/pages/projects/_id/segmentation/index.vue
new file mode 100644
index 00000000..604cd44d
--- /dev/null
+++ b/frontend/pages/projects/_id/segmentation/index.vue
@@ -0,0 +1,321 @@
+<template>
+  <layout-text v-if="image.id">
+    <template #header>
+      <toolbar-laptop
+        :doc-id="image.id"
+        :enable-auto-labeling.sync="enableAutoLabeling"
+        :guideline-text="project.guideline"
+        :is-reviewd="image.isConfirmed"
+        :total="images.count"
+        class="d-none d-sm-block"
+        @click:clear-label="clear"
+        @click:review="confirm"
+      >
+        <v-btn-toggle v-model="labelOption" mandatory class="ms-2">
+          <v-btn icon>
+            <v-icon>{{ mdiFormatListBulleted }}</v-icon>
+          </v-btn>
+          <v-btn icon>
+            <v-icon>{{ mdiText }}</v-icon>
+          </v-btn>
+        </v-btn-toggle>
+      </toolbar-laptop>
+      <toolbar-mobile :total="images.count" class="d-flex d-sm-none" />
+    </template>
+    <template #content>
+      <v-card>
+        <v-card-title>
+          <v-chip-group v-model="selectedLabelIndex" column>
+            <v-chip
+              v-for="item in labels"
+              :key="item.id"
+              :color="item.backgroundColor"
+              filter
+              :text-color="$contrastColor(item.backgroundColor)"
+            >
+              {{ item.text }}
+              <v-avatar
+                v-if="item.suffixKey"
+                right
+                color="white"
+                class="black--text font-weight-bold"
+              >
+                {{ item.suffixKey }}
+              </v-avatar>
+            </v-chip>
+          </v-chip-group>
+        </v-card-title>
+        <v-divider />
+        <v-segmentation
+          :highlight-id="highlightId"
+          :image-url="image.fileUrl"
+          :labels="bboxLabels"
+          :polygons="filteredRegions"
+          :selected-label="selectedLabel"
+          :scale="scale"
+          @add-polygon="add"
+          @delete-polygon="remove"
+          @select-polygon="selectRegion"
+          @update-polygon="update"
+          @update-scale="updateScale"
+        />
+      </v-card>
+    </template>
+    <template #sidebar>
+      <annotation-progress :progress="progress" />
+      <list-metadata :metadata="image.meta" class="mt-4" />
+      <region-list
+        v-if="annotations.length > 0"
+        class="mt-4"
+        :regions="regionList"
+        @change-visibility="changeVisibility"
+        @hover-region="hoverRegion"
+        @unhover-region="unhoverRegion"
+      />
+    </template>
+  </layout-text>
+</template>
+
+<script>
+import _ from 'lodash'
+import { mdiText, mdiFormatListBulleted } from '@mdi/js'
+import { toRefs, useContext } from '@nuxtjs/composition-api'
+import LayoutText from '@/components/tasks/layout/LayoutText'
+import ListMetadata from '@/components/tasks/metadata/ListMetadata'
+import ToolbarLaptop from '@/components/tasks/toolbar/ToolbarLaptop'
+import ToolbarMobile from '@/components/tasks/toolbar/ToolbarMobile'
+import { useLabelList } from '@/composables/useLabelList'
+import AnnotationProgress from '@/components/tasks/sidebar/AnnotationProgress.vue'
+import { VSegmentation } from 'vue-image-annotator'
+import RegionList from '@/components/tasks/image/RegionList.vue'
+
+export default {
+  components: {
+    AnnotationProgress,
+    LayoutText,
+    ListMetadata,
+    RegionList,
+    ToolbarLaptop,
+    ToolbarMobile,
+    VSegmentation
+  },
+  layout: 'workspace',
+
+  validate({ params, query }) {
+    return /^\d+$/.test(params.id) && /^\d+$/.test(query.page)
+  },
+
+  setup() {
+    const { app } = useContext()
+    const { state, getLabelList, shortKeys } = useLabelList(app.$services.categoryType)
+
+    return {
+      ...toRefs(state),
+      getLabelList,
+      shortKeys
+    }
+  },
+
+  data() {
+    return {
+      annotations: [],
+      images: [],
+      project: {},
+      enableAutoLabeling: false,
+      labelOption: 0,
+      mdiText,
+      mdiFormatListBulleted,
+      progress: {},
+      highlightId: null,
+      selectedLabelIndex: undefined,
+      selectedRegion: undefined,
+      visibilities: {},
+      scale: 1
+    }
+  },
+
+  async fetch() {
+    this.images = await this.$services.example.fetchOne(
+      this.projectId,
+      this.$route.query.page,
+      this.$route.query.q,
+      this.$route.query.isChecked
+    )
+    const image = this.images.items[0]
+    if (this.enableAutoLabeling) {
+      await this.autoLabel(image.id)
+    }
+    await this.list(image.id)
+  },
+
+  computed: {
+    projectId() {
+      return this.$route.params.id
+    },
+
+    image() {
+      if (_.isEmpty(this.images) || this.images.items.length === 0) {
+        return {}
+      } else {
+        return this.images.items[0]
+      }
+    },
+
+    bboxLabels() {
+      return this.labels.map((label) => {
+        return {
+          id: label.id,
+          name: label.text,
+          color: label.backgroundColor
+        }
+      })
+    },
+
+    selectedLabel() {
+      if (this.selectedLabelIndex !== undefined) {
+        return this.labels[this.selectedLabelIndex]
+      } else {
+        return undefined
+      }
+    },
+
+    regionList() {
+      return this.annotations.map((annotation) => {
+        return {
+          id: annotation.uuid,
+          category: this.labels.find((label) => annotation.label === label.id).text,
+          color: this.labels.find((label) => annotation.label === label.id).backgroundColor,
+          visibility:
+            annotation.uuid in this.visibilities ? this.visibilities[annotation.uuid] : true
+        }
+      })
+    },
+
+    filteredRegions() {
+      return this.annotations
+        .filter((annotation) => this.visibilities[annotation.uuid] !== false)
+        .map((a) => {
+          return {
+            ...a,
+            id: a.uuid
+          }
+        })
+    }
+  },
+
+  watch: {
+    '$route.query': '$fetch',
+    enableAutoLabeling(val) {
+      if (val) {
+        this.list(this.image.id)
+      }
+    },
+
+    async selectedLabel(newLabel) {
+      if (newLabel !== undefined && !!this.selectedRegion) {
+        this.selectedRegion.label = newLabel.id
+        await this.$services.segmentation.update(
+          this.projectId,
+          this.image.id,
+          this.selectedRegion.id,
+          this.selectedRegion
+        )
+        await this.list(this.image.id)
+      }
+    }
+  },
+
+  async created() {
+    this.getLabelList(this.projectId)
+    this.project = await this.$services.project.findById(this.projectId)
+    this.progress = await this.$services.metrics.fetchMyProgress(this.projectId)
+  },
+
+  methods: {
+    async list(imageId) {
+      this.annotations = await this.$services.segmentation.list(this.projectId, imageId)
+    },
+
+    async remove(id) {
+      delete this.visibilities[id]
+      const segmentation = this.annotations.find((a) => a.uuid === region.id)
+      await this.$services.segmentation.delete(this.projectId, this.image.id, segmentation.id)
+      await this.list(this.image.id)
+    },
+
+    async add(region) {
+      this.visibilities[region.id] = true
+      await this.$services.segmentation.create(
+        this.projectId,
+        this.image.id,
+        region.id,
+        region.label,
+        region.points
+      )
+      await this.list(this.image.id)
+    },
+
+    async update(region) {
+      const segmentation = this.annotations.find((a) => a.uuid === region.id)
+      await this.$services.segmentation.update(
+        this.projectId,
+        this.image.id,
+        segmentation.id,
+        region
+      )
+      await this.list(this.image.id)
+    },
+
+    changeVisibility(regionId, visibility) {
+      this.$set(this.visibilities, regionId, visibility)
+      this.visibilities = Object.assign({}, this.visibilities)
+    },
+
+    async clear() {
+      await this.$services.segmentation.clear(this.projectId, this.image.id)
+      await this.list(this.image.id)
+    },
+
+    async autoLabel(imageId) {
+      try {
+        await this.$services.segmentation.autoLabel(this.projectId, imageId)
+      } catch (e) {
+        console.log(e.response.data.detail)
+      }
+    },
+
+    async updateProgress() {
+      this.progress = await this.$services.metrics.fetchMyProgress(this.projectId)
+    },
+
+    async confirm() {
+      await this.$services.example.confirm(this.projectId, this.image.id)
+      await this.$fetch()
+      this.updateProgress()
+    },
+
+    hoverRegion(regionId) {
+      this.highlightId = regionId
+    },
+
+    unhoverRegion() {
+      this.highlightId = null
+    },
+
+    selectRegion(regionId) {
+      this.selectedRegion = this.annotations.find((r) => r.uuid === regionId)
+      this.selectedLabelIndex = this.labels.findIndex((l) => l.id === this.selectedRegion.label)
+    },
+
+    updateScale(scale) {
+      this.scale = scale
+    }
+  }
+}
+</script>
+
+<style scoped>
+.text-pre-wrap {
+  white-space: pre-wrap !important;
+}
+</style>
diff --git a/frontend/plugins/services.ts b/frontend/plugins/services.ts
index 4e6d3358..11963f86 100644
--- a/frontend/plugins/services.ts
+++ b/frontend/plugins/services.ts
@@ -44,6 +44,8 @@ import { TagApplicationService } from '~/services/application/tag/tagApplication
 import { ApiRelationRepository } from '~/repositories/tasks/sequenceLabeling/apiRelationRepository'
 import { ApiBoundingBoxRepository } from '~/repositories/tasks/boundingBox/apiBoundingBoxRepository'
 import { BoundingBoxApplicationService } from '~/services/application/tasks/boundingBox/boundingBoxApplicationService'
+import { ApiSegmentationRepository } from '~/repositories/tasks/segmentation/apiSegmentationRepository'
+import { SegmentationApplicationService } from '~/services/application/tasks/segmentation/segmentationApplicationService'
 
 export interface Services {
   categoryType: LabelApplicationService
@@ -70,6 +72,7 @@ export interface Services {
   download: DownloadApplicationService
   tag: TagApplicationService
   bbox: BoundingBoxApplicationService
+  segmentation: SegmentationApplicationService
 }
 
 declare module 'vue/types/vue' {
@@ -101,6 +104,7 @@ const plugin: Plugin = (_, inject) => {
   const downloadFormatRepository = new APIDownloadFormatRepository()
   const downloadRepository = new APIDownloadRepository()
   const boundingBoxRepository = new ApiBoundingBoxRepository()
+  const segmentationRepository = new ApiSegmentationRepository()
 
   const categoryType = new LabelApplicationService(new APILabelRepository('category-type'))
   const spanType = new LabelApplicationService(new APILabelRepository('span-type'))
@@ -118,6 +122,7 @@ const plugin: Plugin = (_, inject) => {
     linkRepository
   )
   const bbox = new BoundingBoxApplicationService(boundingBoxRepository)
+  const segmentation = new SegmentationApplicationService(segmentationRepository)
   const seq2seq = new Seq2seqApplicationService(seq2seqRepository)
   const option = new OptionApplicationService(optionRepository)
   const config = new ConfigApplicationService(configRepository)
@@ -154,7 +159,8 @@ const plugin: Plugin = (_, inject) => {
     downloadFormat,
     download,
     tag,
-    bbox
+    bbox,
+    segmentation
   }
   inject('services', services)
 }
diff --git a/frontend/repositories/tasks/segmentation/apiSegmentationRepository.ts b/frontend/repositories/tasks/segmentation/apiSegmentationRepository.ts
new file mode 100644
index 00000000..8ae66a73
--- /dev/null
+++ b/frontend/repositories/tasks/segmentation/apiSegmentationRepository.ts
@@ -0,0 +1,40 @@
+import { AnnotationRepository } from '@/domain/models/tasks/annotationRepository'
+import { SegmentationItem } from '~/domain/models/tasks/segmentation'
+
+export class ApiSegmentationRepository extends AnnotationRepository<SegmentationItem> {
+  constructor() {
+    super(SegmentationItem)
+  }
+
+  async list(projectId: string, exampleId: number): Promise<SegmentationItem[]> {
+    const url = `/projects/${projectId}/examples/${exampleId}/segments`
+    const response = await this.request.get(url)
+    return response.data.map((box: any) => SegmentationItem.valueOf(box))
+  }
+
+  async create(projectId: string, exampleId: number, item: SegmentationItem): Promise<void> {
+    const url = `/projects/${projectId}/examples/${exampleId}/segments`
+    await this.request.post(url, item.toObject())
+  }
+
+  async update(
+    projectId: string,
+    exampleId: number,
+    boxId: number,
+    item: SegmentationItem
+  ): Promise<SegmentationItem> {
+    const url = `/projects/${projectId}/examples/${exampleId}/segments/${boxId}`
+    const response = await this.request.patch(url, item.toObject())
+    return SegmentationItem.valueOf(response.data)
+  }
+
+  async delete(projectId: string, exampleId: number, boxId: number): Promise<void> {
+    const url = `/projects/${projectId}/examples/${exampleId}/segments/${boxId}`
+    await this.request.delete(url)
+  }
+
+  async bulkDelete(projectId: string, exampleId: number, boxIds: number[]): Promise<void> {
+    const url = `/projects/${projectId}/examples/${exampleId}/segments`
+    await this.request.delete(url, { ids: boxIds })
+  }
+}
diff --git a/frontend/services/application/tasks/segmentation/segmentationApplicationService.ts b/frontend/services/application/tasks/segmentation/segmentationApplicationService.ts
new file mode 100644
index 00000000..49e7e9fb
--- /dev/null
+++ b/frontend/services/application/tasks/segmentation/segmentationApplicationService.ts
@@ -0,0 +1,44 @@
+import { AnnotationApplicationService } from '../annotationApplicationService'
+import { SegmentationDTO } from './segmentationData'
+import { ApiSegmentationRepository } from '~/repositories/tasks/segmentation/apiSegmentationRepository'
+import { SegmentationItem } from '~/domain/models/tasks/segmentation'
+
+export class SegmentationApplicationService extends AnnotationApplicationService<SegmentationItem> {
+  constructor(readonly repository: ApiSegmentationRepository) {
+    super(new ApiSegmentationRepository())
+  }
+
+  public async list(projectId: string, exampleId: number): Promise<SegmentationDTO[]> {
+    const items = await this.repository.list(projectId, exampleId)
+    return items.map((item) => new SegmentationDTO(item))
+  }
+
+  public async create(
+    projectId: string,
+    exampleId: number,
+    uuid: string,
+    label: number,
+    points: number[]
+  ): Promise<void> {
+    const item = new SegmentationItem(0, uuid, label, points)
+    try {
+      await this.repository.create(projectId, exampleId, item)
+    } catch (e: any) {
+      console.log(e.response.data.detail)
+    }
+  }
+
+  public async update(
+    projectId: string,
+    exampleId: number,
+    annotationId: number,
+    item: SegmentationDTO
+  ): Promise<void> {
+    const bbox = new SegmentationItem(item.id, item.uuid, item.label, item.points)
+    try {
+      await this.repository.update(projectId, exampleId, annotationId, bbox)
+    } catch (e: any) {
+      console.log(e.response.data.detail)
+    }
+  }
+}
diff --git a/frontend/services/application/tasks/segmentation/segmentationData.ts b/frontend/services/application/tasks/segmentation/segmentationData.ts
new file mode 100644
index 00000000..bb0bdc57
--- /dev/null
+++ b/frontend/services/application/tasks/segmentation/segmentationData.ts
@@ -0,0 +1,15 @@
+import { SegmentationItem } from '~/domain/models/tasks/segmentation'
+
+export class SegmentationDTO {
+  id: number
+  uuid: string
+  label: number
+  points: number[]
+
+  constructor(item: SegmentationItem) {
+    this.id = item.id
+    this.uuid = item.uuid
+    this.label = item.label
+    this.points = item.points
+  }
+}