[gitlab] Remove metadata from non-enriched items

This commit prevents addition of repository label and metadata filter raw to items which are not to be enrihced. This results in creation of empty items which are not uploaded to ElasticSearch. Tests have been added accrodingly. Signed-off-by: Animesh Kumar <[email protected]>
chaoss · Apr 24, 2020 · 0036e06 · 0036e06
1 parent 0574e55
commit 0036e06
Show file tree

Hide file tree

Showing 3 changed files with 80 additions and 6 deletions.
diff --git a/grimoire_elk/enriched/gitlab.py b/grimoire_elk/enriched/gitlab.py
@@ -148,7 +148,8 @@ def get_rich_item(self, item):
         elif item['category'] == 'merge_request':
             rich_item = self.__get_rich_merge(item)
         else:
-            logger.error("[gerrit] rich item not defined for GitLab category {}".format(item['category']))
+            logger.error("[gitlab] rich item not defined for GitLab category {}".format(item['category']))
+            return rich_item
 
         self.add_repository_labels(rich_item)
         self.add_metadata_filter_raw(rich_item)

diff --git a/tests/data/gitlab.json b/tests/data/gitlab.json
@@ -11229,5 +11229,77 @@
         "tag": "https://gitlab.com/fdroid/fdroiddata",
         "timestamp": 1537198073.82518,
         "updated_on": 1526489019.747,
-        "uuid": "11a8636d4c423c4ef45720828ca37ab23e90c31e"}
+        "uuid": "11a8636d4c423c4ef45720828ca37ab23e90c31e"
+    },
+    {
+        "backend_name": "GitLab",
+        "backend_version": "0.5.0",
+        "category": "message",
+        "data": {
+            "assignee": null,
+            "assignees": [],
+            "author": {
+                "avatar_url": "https://secure.gravatar.com/avatar/b8c8a858811dfece044c3818e21bf4f3?s=80&d=identicon",
+                "id": 1,
+                "name": "Timothy Engler",
+                "state": "active",
+                "username": "redfish64",
+                "web_url": "https://gitlab.com/redfish64"
+            },
+            "award_emoji_data": [],
+            "closed_at": null,
+            "confidential": false,
+            "created_at": "2017-03-18T09:52:55.303Z",
+            "description": "Feeder (com.nononsenseapps.feeder) unnecessary require Android 7.0\n\nPlease compile it with lower minSDK.",
+            "discussion_locked": null,
+            "downvotes": 0,
+            "due_date": null,
+            "id": 2,
+            "iid": 2,
+            "labels": [],
+            "milestone": null,
+            "notes_data": [
+                {
+                    "attachment": null,
+                    "author": {
+                        "avatar_url": "https://secure.gravatar.com/avatar/b8c8a858811dfece044c3818e21bf4f3?s=80&d=identicon",
+                        "id": 1,
+                        "name": "Timothy Engler",
+                        "state": "active",
+                        "username": "redfish64",
+                        "web_url": "https://gitlab.com/redfish64"
+                    },
+                    "award_emoji_data": [],
+                    "body": "https://github.com/spacecowboy/Feeder/issues/13",
+                    "created_at": "2017-03-18T18:36:39.757Z",
+                    "id": 1,
+                    "noteable_id": 4756532,
+                    "noteable_iid": 641,
+                    "noteable_type": "Issue",
+                    "system": false,
+                    "updated_at": "2017-03-18T18:36:39.757Z"
+                }
+            ],
+            "project_id": 1,
+            "state": "closed",
+            "time_stats": {
+                "human_time_estimate": null,
+                "human_total_time_spent": null,
+                "time_estimate": 0,
+                "total_time_spent": 0
+            },
+            "title": "Feeder (com.nononsenseapps.feeder) unnecessary require Android 7.0",
+            "updated_at": "2017-03-18T18:36:39.764Z",
+            "upvotes": 0,
+            "user_notes_count": 2,
+            "web_url": "https://gitlab.com/fdroid/fdroiddata/-/issues/641",
+            "weight": null
+        },
+        "origin": "https://gitlab.com/fdroid/fdroiddata",
+        "perceval_version": "0.11.10",
+        "tag": "https://gitlab.com/fdroid/fdroiddata",
+        "timestamp": 1536860519.370875,
+        "updated_on": 1489862199.764,
+        "uuid": "0777f3d8ed29fbfa61d897c9c9eefb519c1bce7d"
+    }
 ]
diff --git a/tests/test_gitlab.py b/tests/test_gitlab.py
@@ -57,7 +57,7 @@ def test_raw_to_enrich(self):
 
         self.assertGreater(result['raw'], 0)
         self.assertGreater(result['enrich'], 0)
-        self.assertEqual(result['raw'], result['enrich'])
+        self.assertEqual(result['raw'] - 1, result['enrich'])
 
         enrich_backend = self.connectors[self.connector][2]()
 
@@ -161,15 +161,16 @@ def test_enrich_repo_labels(self):
 
         for item in self.items:
             eitem = enrich_backend.get_rich_item(item)
-            self.assertIn(REPO_LABELS, eitem)
+            if eitem:
+                self.assertIn(REPO_LABELS, eitem)
 
     def test_raw_to_enrich_sorting_hat(self):
         """Test enrich with SortingHat"""
 
         result = self._test_raw_to_enrich(sortinghat=True)
         self.assertGreater(result['raw'], 0)
         self.assertGreater(result['enrich'], 0)
-        self.assertEqual(result['raw'], result['enrich'])
+        self.assertEqual(result['raw'] - 1, result['enrich'])
 
         enrich_backend = self.connectors[self.connector][2]()
 
@@ -240,7 +241,7 @@ def test_raw_to_enrich_anonymized(self):
 
         self.assertGreater(result['raw'], 0)
         self.assertGreater(result['enrich'], 0)
-        self.assertEqual(result['raw'], result['enrich'])
+        self.assertEqual(result['raw'] - 1, result['enrich'])
 
         enrich_backend = self.connectors[self.connector][2]()