Implemented a better handling of (partially) unsalted hashlists #1657

ci/tests/integration/MaxAgentsTest.class.php

@@ -52,7 +52,7 @@ public function run() {
     HashtopolisTestFramework::log(HashtopolisTestFramework::LOG_INFO, "Running " . $this->getTestName() . "...");
     $this->prepare();
     try {
-      $response = $this->addHashlist(["name" => "NotSecureList", "isSecure" => false]);
+      $response = $this->addHashlist(["name" => "NotSecureList", "isSecure" => false])["hashlist"];
       $hashlistId = $response["hashlistId"];
 
       $this->testTaskMaxAgents($hashlistId);

ci/tests/integration/RuleSplitTest.class.php

@@ -61,7 +61,7 @@ private function testRuleSplit() {
     $file_id2 = $this->getFile('best64.rule');
 
     # Create hashlist
-    $response = $this->addHashlist(["isSecure" => false]);
+    $response = $this->addHashlist(["isSecure" => false])["hashlist"];
     $hashlistId = $response["hashlistId"];
 
     # Create task with rule/wordlist

src/inc/apiv2/common/AbstractBaseAPI.php

@@ -1616,8 +1616,9 @@ static function createJsonResponse(array $data = [], array $links = [], array $i
   /**
    * Get single Resource
    */
-  protected static function getOneResource(object $apiClass, object $object, Request $request, Response $response, int $statusCode = 200): Response {
-    $apiClass->preCommon($request);  
+  protected static function getOneResource(object $apiClass, object $object, Request $request, Response $response, int $statusCode = 200, array|null $creationInformation = null): Response {
+    $apiClass->preCommon($request);
+
     $validExpandables = $apiClass->getExpandables();
     $expands = $apiClass->makeExpandables($request, $validExpandables);
 
@@ -1654,6 +1655,10 @@ protected static function getOneResource(object $apiClass, object $object, Reque
     if ($apiClass->permissionErrors !== null) {
       $metaData["Include errors"] = $apiClass->permissionErrors;
     }
+    if(is_array($creationInformation)) {
+      $metaData["creationInformation"] = $creationInformation;
+    }
+
     // Generate JSON:API GET output
     $ret = self::createJsonResponse($dataResources[0], $links, $includedResources, $metaData);
 

src/inc/apiv2/common/AbstractModelAPI.php

@@ -31,9 +31,13 @@ abstract class AbstractModelAPI extends AbstractBaseAPI {
   abstract static public function getDBAClass();
 
   abstract protected function createObject(array $data): int;
-  
+
   abstract protected function deleteObject(object $object): void;
 
+  protected function createObjectAndGetResult(array $data): array {
+    return [];
+  }
+
   /**
    * Available 'expand' parameters on $object
    */
@@ -1208,11 +1212,20 @@ public function post(Request $request, Response $response, array $args): Respons
 
     // Remove key aliases and sanitize to 'db values and request creation
     $mappedData = $this->unaliasData($attributes, $allFeatures);
-    $pk = $this->createObject($mappedData);
-
+
     // Request object again, since post-modified entries are not reflected into object.
-    $object = $this->getFactory()->get($pk);
-    return self::getOneResource($this, $object, $request, $response, 201);
+    $object = null;
+
+    if (isset($data["getCreationInformation"])) {
+      $creationResult = $this->createObjectAndGetResult($mappedData);
+      $object = $this->getFactory()->get($creationResult["pk"]);
+      return self::getOneResource($this, $object, $request, $response, 201, $creationResult["creationInformation"]);
+    }
+    else {
+      $pk = $this->createObject($mappedData);
+      $object = $this->getFactory()->get($pk);
+      return self::getOneResource($this, $object, $request, $response, 201);
+    }
   }
 
 

src/inc/apiv2/model/HashlistAPI.php

@@ -113,6 +113,15 @@ public function getFormFields(): array {
    * @throws HTException
    */
   protected function createObject(array $data): int {
+    return $this->createObjectAndGetResult($data)["pk"];
+  }
+
+  /**
+   * @throws HttpErrorException
+   * @throws HttpError
+   * @throws HTException
+   */
+  protected function createObjectAndGetResult(array $data): array {
     // Cast to createHashlist compatible upload format
     $dummyPost = [];
     switch ($data["sourceType"]) {
@@ -139,7 +148,7 @@ protected function createObject(array $data): int {
       }
     }
 
-    $hashlist = HashlistUtils::createHashlist(
+    $hashlistData = HashlistUtils::createHashlist(
       $data[Hashlist::HASHLIST_NAME],
       $data[Hashlist::IS_SALTED],
       $data[Hashlist::IS_SECRET],
@@ -159,11 +168,14 @@ protected function createObject(array $data): int {
 
     // Modify fields not set on hashlist creation
     if (array_key_exists("notes", $data)) {
-      HashlistUtils::editNotes($hashlist->getId(), $data["notes"], $this->getCurrentUser());
-    };
-    HashlistUtils::setArchived($hashlist->getId(), $data[UQueryHashlist::HASHLIST_IS_ARCHIVED], $this->getCurrentUser());
+      HashlistUtils::editNotes($hashlistData["hashlist"]->getId(), $data["notes"], $this->getCurrentUser());
+    }
+    HashlistUtils::setArchived($hashlistData["hashlist"]->getId(), $data[UQueryHashlist::HASHLIST_IS_ARCHIVED], $this->getCurrentUser());
 
-    return $hashlist->getId();
+    $creationResult["pk"] = $hashlistData["hashlist"]->getId();
+    $creationResult["creationInformation"] = $hashlistData["statistics"];
+
+    return $creationResult;
   }
 
   /**

src/inc/defines/DHashlistStatistics.php

@@ -0,0 +1,11 @@
+<?php
+
+namespace Hashtopolis\inc\defines;
+
+class DHashlistStatistics {
+  const UPLOADED_TOTAL_LINES                        = "uploadedTotalLines";
+  const UPLOADED_EMPTY_LINES                        = "uploadedEmptyLines";
+  const UPLOADED_VALID_HASHES                       = "uploadedValidHashes";
+  const UPLOADED_VALID_HASHES_WITHOUT_EXPECTED_SALT = "uploadedValidHashesWithoutExpectedSalt";
+  const UPLOADED_INVALID_HASHES                     = "uploadedInvalidHashes";
+}

src/inc/handlers/HashlistHandler.php

@@ -104,7 +104,7 @@ public function handle($action) {
             AccessControl::getInstance()->getUser(),
             (isset($_POST["useBrain"]) && intval($_POST["useBrain"]) == 1) ? 1 : 0,
             (isset($_POST['brain-features'])) ? intval($_POST['brain-features']) : 0
-          );
+          )["hashlist"];
           header("Location: hashlists.php?id=" . $hashlist->getId());
           die();
         case DHashlistAction::CREATE_SUPERHASHLIST:

src/inc/user_api/UserAPIHashlist.php

@@ -286,7 +286,7 @@ private function createHashlist($QUERY) {
       $this->user,
       $QUERY[UQueryHashlist::HASHLIST_USE_BRAIN],
       $QUERY[UQueryHashlist::HASHLIST_BRAIN_FEATURES]
-    );
+    )["hashlist"];
     $this->sendResponse(array(
         UResponseHashlist::SECTION => $QUERY[UQuery::SECTION],
         UResponseHashlist::REQUEST => $QUERY[UQuery::REQUEST],

src/inc/utils/HashlistUtils.php

@@ -31,6 +31,7 @@
 use Hashtopolis\inc\defines\DDirectories;
 use Hashtopolis\inc\defines\DFileType;
 use Hashtopolis\inc\defines\DHashlistFormat;
+use Hashtopolis\inc\defines\DHashlistStatistics;
 use Hashtopolis\inc\defines\DLogEntry;
 use Hashtopolis\inc\defines\DNotificationObjectType;
 use Hashtopolis\inc\defines\DNotificationType;
@@ -782,7 +783,8 @@ public static function export($hashlistId, $user) {
    * @param User $user
    * @param int $brainId
    * @param int $brainFeatures
-   * @return Hashlist
+   * @param boolean $writeResultsToNotes
+   * @return array
    * @throws HTException
    */
   public static function createHashlist($name, $isSalted, $isSecret, $isHexSalted, $separator, $format, $hashtype, $saltSeparator, $accessGroupId, $source, $post, $files, $user, $brainId, $brainFeatures) {
@@ -851,33 +853,62 @@ public static function createHashlist($name, $isSalted, $isSecret, $isHexSalted,
       Factory::getAgentFactory()->getDB()->rollback();
       throw new HttpError("Hashlist has too many lines!");
     }
+
     $file = fopen($tmpfile, "rb");
     if (!$file) {
       Factory::getAgentFactory()->getDB()->rollback();
       throw new HttpError("Failed to open file!");
     }
+
+    if ($format == DHashlistFormat::PLAIN && $salted) {
+      // find out if the file contains a salt separator at all
+      rewind($file);
+
+      $saltSeparatorFound = false;
+      while (($currentLine = fgets($file)) !== false) {
+        if (strpos($currentLine, $saltSeparator) !== false) {
+          $saltSeparatorFound = true;
+          break;
+        }
+      }
+
+      if ($saltSeparatorFound === false) {
+        fclose($file);
+        unlink($tmpfile);
+        Factory::getAgentFactory()->getDB()->rollback();
+
+        throw new HttpError("Salted hashes separator not found at all in the hashlist! Hashlist not created.");
+      }
+    }
+    else {
+      $saltSeparator = "";
+    }
+
+    Factory::getAgentFactory()->getDB()->commit();
+
     $added = 0;
     $preFound = 0;
+    $hashlistStatistics = [];
+    $hashlistStatistics[DHashlistStatistics::UPLOADED_TOTAL_LINES] = 0;
+    $hashlistStatistics[DHashlistStatistics::UPLOADED_EMPTY_LINES] = 0;
+    $hashlistStatistics[DHashlistStatistics::UPLOADED_VALID_HASHES] = 0;
+    $hashlistStatistics[DHashlistStatistics::UPLOADED_VALID_HASHES_WITHOUT_EXPECTED_SALT] = 0;
+    $hashlistStatistics[DHashlistStatistics::UPLOADED_INVALID_HASHES] = 0;
 
     switch ($format) {
       case DHashlistFormat::PLAIN:
-        if ($salted) {
-          // find out if the first line contains field separator
-          rewind($file);
-          $bufline = stream_get_line($file, 1024);
-          if (strpos($bufline, $saltSeparator) === false) {
-            throw new HttpError("Salted hashes separator not found in file!");
-          }
-        }
-        else {
-          $saltSeparator = "";
-        }
         rewind($file);
+
+        Factory::getAgentFactory()->getDB()->beginTransaction();
         $values = array();
         $bufferCount = 0;
+
         while (!feof($file)) {
           $line = trim(fgets($file));
+          $hashlistStatistics[DHashlistStatistics::UPLOADED_TOTAL_LINES]++;
+
           if (strlen($line) == 0) {
+            $hashlistStatistics[DHashlistStatistics::UPLOADED_EMPTY_LINES]++;
             continue;
           }
           $hash = $line;
@@ -888,8 +919,12 @@ public static function createHashlist($name, $isSalted, $isSecret, $isHexSalted,
               $hash = substr($line, 0, $pos);
               $salt = substr($line, $pos + 1);
             }
+            else {
+              $hashlistStatistics[DHashlistStatistics::UPLOADED_VALID_HASHES_WITHOUT_EXPECTED_SALT]++;
+            }
           }
           if (strlen($hash) == 0) {
+            $hashlistStatistics[DHashlistStatistics::UPLOADED_EMPTY_LINES]++;
             continue;
           }
           //TODO: check hash length here
@@ -906,13 +941,15 @@ public static function createHashlist($name, $isSalted, $isSecret, $isHexSalted,
               }
             }
           }
+
           if ($found == null) {
             $values[] = new Hash(null, $hashlist->getId(), $hash, $salt, "", 0, null, 0, 0);
           }
           else {
             $values[] = new Hash(null, $hashlist->getId(), $hash, $salt, $found->getPlaintext(), time(), null, 1, 0);
             $preFound++;
           }
+
           $bufferCount++;
           if ($bufferCount >= 5000) {
             $result = Factory::getHashFactory()->massSave($values);
@@ -921,27 +958,31 @@ public static function createHashlist($name, $isSalted, $isSecret, $isHexSalted,
             $bufferCount = 0;
           }
         }
+
         if (sizeof($values) > 0) {
           $result = Factory::getHashFactory()->massSave($values);
           $added += $result->rowCount();
         }
+
         fclose($file);
         unlink($tmpfile);
-        Factory::getHashlistFactory()->mset($hashlist, [Hashlist::HASH_COUNT => $added, Hashlist::CRACKED => $preFound]);
-        Util::createLogEntry("User", $user->getId(), DLogEntry::INFO, "New Hashlist created: " . $hashlist->getHashlistName());
-
-        NotificationHandler::checkNotifications(DNotificationType::NEW_HASHLIST, new DataSet(array(DPayloadKeys::HASHLIST => $hashlist)));
+
         break;
       case DHashlistFormat::WPA:
         $added = 0;
         $values = [];
+
         while (!feof($file)) {
+          $hashlistStatistics[DHashlistStatistics::UPLOADED_TOTAL_LINES]++;
+
           if ($hashlist->getHashTypeId() == 2500) { // HCCAPX hashes
             $data = fread($file, 393);
             if (strlen($data) == 0) {
+              $hashlistStatistics[DHashlistStatistics::UPLOADED_INVALID_HASHES]++;
               break;
             }
             if (strlen($data) != 393) {
+              $hashlistStatistics[DHashlistStatistics::UPLOADED_INVALID_HASHES]++;
               UI::printError("ERROR", "Data file only contains " . strlen($data) . " bytes!");
             }
             // get the SSID
@@ -969,11 +1010,13 @@ public static function createHashlist($name, $isSalted, $isSecret, $isHexSalted,
             $mac_cli = Util::bintohex($mac_cli);
             $hash = new HashBinary(null, $hashlist->getId(), $mac_ap . SConfig::getInstance()->getVal(DConfig::FIELD_SEPARATOR) . $mac_cli . SConfig::getInstance()->getVal(DConfig::FIELD_SEPARATOR) . Util::bintohex($network), Util::bintohex($data), null, 0, null, 0, 0);
             Factory::getHashBinaryFactory()->save($hash);
+            $hashlistStatistics[DHashlistStatistics::UPLOADED_VALID_HASHES]++;
             $added++;
           }
           else { // PMKID hashes
             $line = trim(fgets($file));
             if (strlen($line) == 0) {
+              $hashlistStatistics[DHashlistStatistics::UPLOADED_EMPTY_LINES]++;
               continue;
             }
             if (strpos($line, "*") !== false) {
@@ -992,33 +1035,49 @@ public static function createHashlist($name, $isSalted, $isSecret, $isHexSalted,
             }
             $hash = new HashBinary(null, $hashlist->getId(), $identification, Util::bintohex($line . "\n"), null, 0, null, 0, 0);
             Factory::getHashBinaryFactory()->save($hash);
+            $hashlistStatistics[DHashlistStatistics::UPLOADED_VALID_HASHES]++;
             $added++;
           }
         }
         fclose($file);
         unlink($tmpfile);
 
-        Factory::getHashlistFactory()->set($hashlist, Hashlist::HASH_COUNT, $added);
-        Util::createLogEntry("User", $user->getId(), DLogEntry::INFO, "New Hashlist created: " . $hashlist->getHashlistName());
-
-        NotificationHandler::checkNotifications(DNotificationType::NEW_HASHLIST, new DataSet(array(DPayloadKeys::HASHLIST => $hashlist)));
         break;
       case DHashlistFormat::BINARY:
+        $added = 0;
+
         if (!feof($file)) {
           $data = fread($file, Util::filesize($tmpfile));
           $hash = new HashBinary(null, $hashlist->getId(), "", Util::bintohex($data), "", 0, null, 0, 0);
           Factory::getHashBinaryFactory()->save($hash);
+          $hashlistStatistics[DHashlistStatistics::UPLOADED_VALID_HASHES]++;
+          $added++;
         }
+
         fclose($file);
         unlink($tmpfile);
-        Factory::getHashlistFactory()->set($hashlist, Hashlist::HASH_COUNT, 1);
-        Util::createLogEntry("User", $user->getId(), DLogEntry::INFO, "New Hashlist created: " . $hashlist->getHashlistName());
 
-        NotificationHandler::checkNotifications(DNotificationType::NEW_HASHLIST, new DataSet(array(DPayloadKeys::HASHLIST => $hashlist)));
         break;
     }
+
+    if ($added === 0) {
+      Factory::getAgentFactory()->getDB()->rollback();
+      Factory::getHashlistFactory()->delete($hashlist);
+      Factory::getAgentFactory()->getDB()->commit();
+      throw new HttpError("No valid hashes found! Hashlist not created.");
+    }
+
+    Factory::getHashlistFactory()->set($hashlist, Hashlist::HASH_COUNT, $added);
+    if($format == DHashlistFormat::PLAIN) {
+      Factory::getHashlistFactory()->set($hashlist, Hashlist::CRACKED, $preFound);
+    }
+
+    Util::createLogEntry("User", $user->getId(), DLogEntry::INFO, "New Hashlist created: " . $hashlist->getHashlistName() . ". Total lines: " . $hashlistStatistics[DHashlistStatistics::UPLOADED_TOTAL_LINES] . " Empty lines: " . $hashlistStatistics[DHashlistStatistics::UPLOADED_EMPTY_LINES] . " Valid hashes: " . $hashlistStatistics[DHashlistStatistics::UPLOADED_VALID_HASHES] . " Valid hashes without expected salt: " . $hashlistStatistics[DHashlistStatistics::UPLOADED_VALID_HASHES_WITHOUT_EXPECTED_SALT] . " Invalid hashes: " . $hashlistStatistics[DHashlistStatistics::UPLOADED_INVALID_HASHES]);
+
+    NotificationHandler::checkNotifications(DNotificationType::NEW_HASHLIST, new DataSet(array(DPayloadKeys::HASHLIST => $hashlist)));
+
     Factory::getAgentFactory()->getDB()->commit();
-    return $hashlist;
+    return ["hashlist" => $hashlist, "statistics" => $hashlistStatistics];
   }
 
   /**