Merge branch 'release/3.1.7'

CHANGELOG.md

@@ -1,5 +1,19 @@
 # Change Log
 
+## [3.1.7](https://github.com/TheHive-Project/Cortex/milestone/34) (2022-10-07)
+
+**Implemented enhancements:**
+
+- Prevent invalid analyzer when a new version is available [\#426](https://github.com/TheHive-Project/Cortex/issues/426)
+- Display job parameters in the report page [\#430](https://github.com/TheHive-Project/Cortex/issues/430)
+- An error in docker pull should not stop the analysis [\#431](https://github.com/TheHive-Project/Cortex/issues/431)
+- Improve catalog parsing [\#432](https://github.com/TheHive-Project/Cortex/issues/432)
+
+**Closed issues:**
+
+- [BUG] CA Certs parameter can't be set back to null [\#377](https://github.com/TheHive-Project/Cortex/issues/377)
+- [FR] See user and organisation who triggered a responder in Cortex WebUI [\#394](https://github.com/TheHive-Project/Cortex/issues/394)
+
 ## [3.1.6](https://github.com/TheHive-Project/Cortex/milestone/32) (2022-06-22)
 
 **Fixed bugs:**

app/org/thp/cortex/controllers/StatusCtrl.scala

@@ -1,57 +1,71 @@
 package org.thp.cortex.controllers
 
-import scala.concurrent.ExecutionContext
-
+import scala.concurrent.{ExecutionContext, Future}
 import play.api.Configuration
 import play.api.http.Status
 import play.api.libs.json.Json.toJsFieldJsValueWrapper
-import play.api.libs.json.{JsBoolean, JsString, Json}
+import play.api.libs.json.{JsBoolean, JsNull, JsString, Json}
 import play.api.mvc.{AbstractController, Action, AnyContent, ControllerComponents}
-
 import com.sksamuel.elastic4s.ElasticDsl
+import org.elastic4play.controllers.Authenticated
+
 import javax.inject.{Inject, Singleton}
 import org.elasticsearch.client.Node
-import org.thp.cortex.models.Worker
-
+import org.thp.cortex.models.{Roles, Worker, WorkerType}
 import org.elastic4play.services.AuthSrv
 import org.elastic4play.services.auth.MultiAuthSrv
+import org.thp.cortex.services.WorkerSrv
 
 @Singleton
 class StatusCtrl @Inject() (
     configuration: Configuration,
     authSrv: AuthSrv,
+    workerSrv: WorkerSrv,
     components: ControllerComponents,
+    authenticated: Authenticated,
     implicit val ec: ExecutionContext
 ) extends AbstractController(components)
     with Status {
 
   private[controllers] def getVersion(c: Class[_]) = Option(c.getPackage.getImplementationVersion).getOrElse("SNAPSHOT")
 
-  def get: Action[AnyContent] = Action {
-    Ok(
-      Json.obj(
-        "versions" -> Json.obj(
-          "Cortex"               -> getVersion(classOf[Worker]),
-          "Elastic4Play"         -> getVersion(classOf[AuthSrv]),
-          "Play"                 -> getVersion(classOf[AbstractController]),
-          "Elastic4s"            -> getVersion(classOf[ElasticDsl]),
-          "ElasticSearch client" -> getVersion(classOf[Node])
-        ),
-        "config" -> Json.obj(
-          "protectDownloadsWith" -> configuration.get[String]("datastore.attachment.password"),
-          "authType" -> (authSrv match {
-            case multiAuthSrv: MultiAuthSrv =>
-              multiAuthSrv.authProviders.map { a =>
-                JsString(a.name)
-              }
-            case _ => JsString(authSrv.name)
-          }),
-          "capabilities" -> authSrv.capabilities.map(c => JsString(c.toString)),
-          "ssoAutoLogin" -> JsBoolean(configuration.getOptional[Boolean]("auth.sso.autologin").getOrElse(false))
+  def get: Action[AnyContent] =
+    Action {
+      Ok(
+        Json.obj(
+          "versions" -> Json.obj(
+            "Cortex"               -> getVersion(classOf[Worker]),
+            "Elastic4Play"         -> getVersion(classOf[AuthSrv]),
+            "Play"                 -> getVersion(classOf[AbstractController]),
+            "Elastic4s"            -> getVersion(classOf[ElasticDsl]),
+            "ElasticSearch client" -> getVersion(classOf[Node])
+          ),
+          "config" -> Json.obj(
+            "protectDownloadsWith" -> configuration.get[String]("datastore.attachment.password"),
+            "authType" -> (authSrv match {
+              case multiAuthSrv: MultiAuthSrv =>
+                multiAuthSrv.authProviders.map { a =>
+                  JsString(a.name)
+                }
+              case _ => JsString(authSrv.name)
+            }),
+            "capabilities" -> authSrv.capabilities.map(c => JsString(c.toString)),
+            "ssoAutoLogin" -> JsBoolean(configuration.getOptional[Boolean]("auth.sso.autologin").getOrElse(false))
+          )
         )
       )
-    )
-  }
+    }
+
+  def getAlerts: Action[AnyContent] =
+    authenticated(Roles.read).async { implicit request =>
+      workerSrv.obsoleteWorkersForUser(request.userId).map { obsoleteWorkers =>
+        val (obsoleteAnalyzers, obsoleteResponders) = obsoleteWorkers.partition(_.tpe() == WorkerType.analyzer)
+        val alerts =
+          (if (obsoleteAnalyzers.nonEmpty) List("ObsoleteAnalyzers") else Nil) :::
+            (if (obsoleteResponders.nonEmpty) List("ObsoleteResponders") else Nil)
+        Ok(Json.toJson(alerts))
+      }
+    }
 
   def health: Action[AnyContent] = TODO
 }

app/org/thp/cortex/models/WorkerDefinition.scala

@@ -110,7 +110,23 @@ object WorkerDefinition {
 
   def reads(workerType: WorkerType.Type): Reads[List[WorkerDefinition]] = {
     val reads = singleReads(workerType)
-    reads.map(List(_)) orElse Reads.list(reads)
+    implicitly[Reads[JsArray]]
+    reads.map(List(_)) orElse JsArrayReads.map(array =>
+      array
+        .value
+        .toList
+        .flatMap { js =>
+          reads
+            .reads(js)
+            .fold(
+              invalid => {
+                logger.warn(s"The catalog contains an invalid entry\n  entry:$js\n  $invalid")
+                Seq.empty
+              },
+              Seq(_)
+            )
+        }
+    )
   }
 
   implicit val writes: Writes[WorkerDefinition] = Writes[WorkerDefinition] { workerDefinition =>

app/org/thp/cortex/services/DockerJobRunnerSrv.scala

@@ -62,7 +62,7 @@ class DockerJobRunnerSrv(
       ec: ExecutionContext
   ): Try[Unit] = {
     import scala.collection.JavaConverters._
-    if (autoUpdate) client.pull(dockerImage)
+    if (autoUpdate) Try(client.pull(dockerImage))
     //    ContainerConfig.builder().addVolume()
     val hostConfigBuilder = HostConfig.builder()
     config.getOptional[Seq[String]]("docker.container.capAdd").map(_.asJava).foreach(hostConfigBuilder.capAdd)

app/org/thp/cortex/services/JobRunnerSrv.scala

@@ -129,7 +129,7 @@ class JobRunnerSrv @Inject() (
           .deepMerge(worker.config)
           .deepMerge(proxy_http)
           .deepMerge(proxy_https)
-        (worker.config \ "cacerts").asOpt[String].foreach { cacerts =>
+        (worker.config \ "cacerts").asOpt[String].filterNot(_.trim.isEmpty).foreach { cacerts =>
           val cacertsFile = jobFolder.resolve("input").resolve("cacerts")
           Files.write(cacertsFile, cacerts.getBytes)
         }

app/org/thp/cortex/services/WorkerSrv.scala

@@ -1,28 +1,26 @@
 package org.thp.cortex.services
 
-import java.net.URL
-import java.nio.file.{Files, Path, Paths}
-
-import scala.collection.JavaConverters._
-import scala.concurrent.{ExecutionContext, Future}
-import scala.io.Codec
-import scala.util.{Failure, Success, Try}
-
-import play.api.libs.json.{JsArray, JsObject, JsString, Json}
-import play.api.{Configuration, Logger}
-
 import akka.NotUsed
 import akka.stream.Materializer
 import akka.stream.scaladsl.{Sink, Source}
-import javax.inject.{Inject, Provider, Singleton}
-import org.scalactic.Accumulation._
-import org.scalactic._
-import org.thp.cortex.models._
-
 import org.elastic4play._
 import org.elastic4play.controllers.{Fields, StringInputValue}
 import org.elastic4play.database.ModifyConfig
+import org.elastic4play.services.QueryDSL.any
 import org.elastic4play.services._
+import org.scalactic.Accumulation._
+import org.scalactic._
+import org.thp.cortex.models._
+import play.api.libs.json.{JsObject, JsString, Json}
+import play.api.{Configuration, Logger}
+
+import java.net.URL
+import java.nio.file.{Files, Path, Paths}
+import javax.inject.{Inject, Provider, Singleton}
+import scala.collection.JavaConverters._
+import scala.concurrent.{ExecutionContext, Future}
+import scala.io.Codec
+import scala.util.{Failure, Success, Try}
 
 @Singleton
 class WorkerSrv @Inject() (
@@ -128,21 +126,23 @@ class WorkerSrv @Inject() (
   private def find(queryDef: QueryDef, range: Option[String], sortBy: Seq[String]): (Source[Worker, NotUsed], Future[Long]) =
     findSrv[WorkerModel, Worker](workerModel, queryDef, range, sortBy)
 
-  def rescan(): Unit = {
-    import org.elastic4play.services.QueryDSL._
+  def rescan(): Unit =
     scan(
       analyzersURLs.map(_    -> WorkerType.analyzer) ++
         respondersURLs.map(_ -> WorkerType.responder)
-    ).onComplete { _ =>
-      userSrv.inInitAuthContext { implicit authContext =>
-        find(any, Some("all"), Nil)._1.runForeach { worker =>
-          workerMap.get(worker.workerDefinitionId()) match {
-            case Some(wd) => update(worker, Fields.empty.set("dataTypeList", Json.toJson(wd.dataTypeList)))
-            case None     => update(worker, Fields.empty.set("dataTypeList", JsArray.empty))
-          }
-        }
-      }
+    )
+
+  def obsoleteWorkersForUser(userId: String): Future[Seq[Worker]] =
+    userSrv.get(userId).flatMap { user =>
+      obsoleteWorkersForOrganization(user.organization())
     }
+
+  def obsoleteWorkersForOrganization(organizationId: String): Future[Seq[Worker]] = {
+    import org.elastic4play.services.QueryDSL._
+    find(withParent("organization", organizationId), Some("all"), Nil)
+      ._1
+      .filterNot(worker => workerMap.contains(worker.workerDefinitionId()))
+      .runWith(Sink.seq)
   }
 
   def scan(workerUrls: Seq[(String, WorkerType.Type)]): Future[Unit] = {
@@ -250,7 +250,7 @@ class WorkerSrv @Inject() (
               .set("command", workerDefinition.command.map(p => JsString(p.toString)))
               .set("url", workerDefinition.url)
               .set("license", workerDefinition.license)
-              .set("baseConfig", workerDefinition.baseConfiguration.map(JsString.apply))
+              .set("baseConfig", workerDefinition.baseConfiguration.fold(JsString(workerDefinition.name))(JsString.apply))
               .set("configuration", cfg.toString)
               .set("type", workerDefinition.tpe.toString)
               .addIfAbsent("dataTypeList", StringInputValue(workerDefinition.dataTypeList))

conf/application.sample

@@ -32,7 +32,7 @@ search {
   #}
 
   ## Authentication configuration
-  #username = ""
+  #user = ""
   #password = ""
 
   ## SSL configuration

conf/routes

@@ -13,6 +13,7 @@ POST          /api/ssoLogin                                         org.thp.cort
 ###################
 # API used by TheHive
 GET           /api/status                                           org.thp.cortex.controllers.StatusCtrl.get
+GET           /api/alert                                            org.thp.cortex.controllers.StatusCtrl.getAlerts
 GET           /api/analyzer                                         org.thp.cortex.controllers.AnalyzerCtrl.find
 POST          /api/analyzer/_search                                 org.thp.cortex.controllers.AnalyzerCtrl.find
 GET           /api/analyzer/:id                                     org.thp.cortex.controllers.AnalyzerCtrl.get(id)

version.sbt

@@ -1 +1 @@
-ThisBuild / version := "3.1.6-1"
+ThisBuild / version := "3.1.7-1"

www/.nvmrc

@@ -0,0 +1 @@
+v12.18

www/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cortex",
-  "version": "3.1.6",
+  "version": "3.1.7",
   "description": "A powerfull observable analysis engine",
   "license": "AGPL-3.0-or-later",
   "homepage": "https://github.com/TheHive-Project/Cortex",

www/src/app/components/header/header.component.js

@@ -13,22 +13,24 @@ class HeaderController {
   constructor(
     $state,
     $log,
-    $q,
     $uibModal,
+    $scope,
     AuthService,
     AnalyzerService,
-    NotificationService
+    NotificationService,
+    AlertService
   ) {
     'ngInject';
 
     this.$state = $state;
     this.$log = $log;
     this.$uibModal = $uibModal;
-    this.$q = $q;
+    this.$scope = $scope;
 
     this.AuthService = AuthService;
     this.AnalyzerService = AnalyzerService;
     this.NotificationService = NotificationService;
+    this.AlertService = AlertService;
   }
 
   logout() {
@@ -44,6 +46,11 @@ class HeaderController {
   $onInit() {
     this.isOrgAdmin = this.AuthService.isOrgAdmin(this.main.currentUser);
     this.isSuperAdmin = this.AuthService.isSuperAdmin(this.main.currentUser);
+
+    this.AlertService.startUpdate();
+    this.$scope.$on('$destroy', () => {
+      this.AlertService.stopUpdate();
+    });
   }
 
   newAnalysis() {
@@ -67,8 +74,8 @@ class HeaderController {
         if (!_.isString(err)) {
           this.NotificationService.error(
             err.data.message ||
-              `An error occurred: ${err.statusText}` ||
-              'An unexpected error occurred'
+            `An error occurred: ${err.statusText}` ||
+            'An unexpected error occurred'
           );
         }
       });

www/src/app/components/header/header.html

@@ -47,7 +47,8 @@
         </li>
         <li ui-sref-active="active" require-roles="orgadmin">
           <a ui-sref="main.organization({id: $ctrl.main.currentUser.organization})">
-            <i class="fa fa-file-text"></i>
+            <i class="fa fa-file-text" ng-if="$ctrl.AlertService.isEmpty()"></i>
+            <i class="fa fa-file-text notif-badge" ng-if="$ctrl.AlertService.nonEmpty()"></i>
             <strong>Organization</strong>
           </a>
         </li>
@@ -116,4 +117,4 @@
       </ul>
     </div>
   </div>
-</nav>
+</nav>

www/src/app/components/header/header.scss

@@ -1,14 +1,16 @@
 .navbar {
   .avatar.avatar-xs {
     line-height: 20px;
+
     span {
       line-height: 20px;
       font-weight: bold;
     }
+
     .avatar-icon {
       margin-top: -5px;
     }
-  }   
+  }
 }
 
 .profile-dropdown {
@@ -18,4 +20,19 @@
       right: 0;
     }
   }
+}
+
+.notif-badge {
+  position: relative;
+}
+
+.notif-badge:after {
+  position: absolute;
+  content: "";
+  width: 10px;
+  height: 10px;
+  left: -5px;
+  top: -5px;
+  background-color: rgb(240, 43, 43);
+  border-radius: 50%;
 }