GameDatabase: Switch to YAML

.github/workflows/gamedb-lint.yml

@@ -3,13 +3,15 @@ name: GameDB Lint
 on:
   pull_request:
     paths:
-      - 'data/resources/gamedb.json'
+      - 'data/resources/gamedb.yaml'
+      - 'data/resources/discdb.yaml'
   push:
     branches:
       - master
       - dev
     paths:
-      - 'data/resources/gamedb.json'
+      - 'data/resources/gamedb.yaml'
+      - 'data/resources/discdb.yaml'
   workflow_dispatch:
 
 jobs:
@@ -25,9 +27,12 @@ jobs:
       shell: bash
       run: |
         sudo apt-get update
-        sudo apt-get -y install python3-demjson
+        sudo apt-get -y install yamllint
 
     - name: Check GameDB
       shell: bash
-      run: |
-        jsonlint -s data/resources/gamedb.json
+      run: yamllint -c extras/yamllint-config.yaml -s -f github data/resources/gamedb.yaml
+
+    - name: Check DiscDB
+      shell: bash
+      run: yamllint -c extras/yamllint-config.yaml -s -f github data/resources/discdb.yaml

extras/yamllint-config.yaml

@@ -0,0 +1,16 @@
+extends: default
+
+rules:
+  line-length:
+    max: 200
+  indentation:
+    spaces: 2
+    indent-sequences: true
+  document-start:
+    present: false
+  document-end:
+    present: false
+  comments:
+    require-starting-space: true
+    min-spaces-from-content: 1
+

src/core/CMakeLists.txt

@@ -131,7 +131,7 @@ target_precompile_headers(core PRIVATE "pch.h")
 target_include_directories(core PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/..")
 target_include_directories(core PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/..")
 target_link_libraries(core PUBLIC Threads::Threads common util ZLIB::ZLIB)
-target_link_libraries(core PRIVATE stb xxhash imgui rapidjson rcheevos)
+target_link_libraries(core PRIVATE stb xxhash imgui rapidyaml rcheevos)
 
 if(CPU_ARCH_X64)
   target_compile_definitions(core PUBLIC "ENABLE_RECOMPILER=1" "ENABLE_NEWREC=1" "ENABLE_MMAP_FASTMEM=1")

src/core/core.props

@@ -10,7 +10,11 @@
       <PreprocessorDefinitions Condition="('$(Platform)'=='x64' Or '$(Platform)'=='ARM64')">ENABLE_MMAP_FASTMEM=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <PreprocessorDefinitions Condition="('$(Platform)'=='x64' Or '$(Platform)'=='ARM64')">ENABLE_NEWREC=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
 
-      <AdditionalIncludeDirectories>%(AdditionalIncludeDirectories);$(SolutionDir)dep\zlib\include;$(SolutionDir)dep\rcheevos\include;$(SolutionDir)dep\rapidjson\include;$(SolutionDir)dep\discord-rpc\include</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>%(AdditionalIncludeDirectories);$(SolutionDir)dep\zlib\include;$(SolutionDir)dep\rcheevos\include;$(SolutionDir)dep\discord-rpc\include</AdditionalIncludeDirectories>
+
+      <PreprocessorDefinitions>%(PreprocessorDefinitions);C4_NO_DEBUG_BREAK=1</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>%(AdditionalIncludeDirectories);$(SolutionDir)dep\rapidyaml\include;$(SolutionDir)dep\rapidjson\include</AdditionalIncludeDirectories>
+
       <AdditionalIncludeDirectories Condition="'$(Platform)'!='ARM64'">%(AdditionalIncludeDirectories);$(SolutionDir)dep\rainterface</AdditionalIncludeDirectories>
       
       <AdditionalIncludeDirectories Condition="'$(Platform)'=='x64'">%(AdditionalIncludeDirectories);$(SolutionDir)dep\xbyak\xbyak</AdditionalIncludeDirectories>

src/core/core.vcxproj

@@ -172,6 +172,9 @@
     <ProjectReference Include="..\..\dep\rainterface\rainterface.vcxproj" Condition="'$(Platform)'!='ARM64'">
       <Project>{e4357877-d459-45c7-b8f6-dcbb587bb528}</Project>
     </ProjectReference>
+    <ProjectReference Include="..\..\dep\rapidyaml\rapidyaml.vcxproj">
+      <Project>{1ad23a8a-4c20-434c-ae6b-0e07759eeb1e}</Project>
+    </ProjectReference>
     <ProjectReference Include="..\..\dep\rcheevos\rcheevos.vcxproj">
       <Project>{4ba0a6d4-3ae1-42b2-9347-096fd023ff64}</Project>
     </ProjectReference>

src/core/game_database.cpp

@@ -1,4 +1,4 @@
-// SPDX-FileCopyrightText: 2019-2023 Connor McLaughlin <stenzek@gmail.com>
+// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
 // SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
 
 #include "game_database.h"
@@ -16,22 +16,18 @@
 #include "common/string_util.h"
 #include "common/timer.h"
 
-#include "rapidjson/document.h"
-#include "rapidjson/error/en.h"
+#include "ryml.hpp"
 
 #include <iomanip>
 #include <memory>
 #include <optional>
 #include <sstream>
+#include <type_traits>
 
 #include "IconsFontAwesome5.h"
 
 Log_SetChannel(GameDatabase);
 
-#ifdef _WIN32
-#include "common/windows_headers.h"
-#endif
-
 namespace GameDatabase {
 
 enum : u32
@@ -46,35 +42,51 @@ static const Entry* GetEntryForId(const std::string_view& code);
 static bool LoadFromCache();
 static bool SaveToCache();
 
-static bool LoadGameDBJson();
-static bool ParseJsonEntry(Entry* entry, const rapidjson::Value& value);
-static bool ParseJsonCodes(u32 index, const rapidjson::Value& value);
+static void SetRymlCallbacks();
+static bool LoadGameDBYaml();
+static bool ParseYamlEntry(Entry* entry, const ryml::ConstNodeRef& value);
+static bool ParseYamlCodes(u32 index, const ryml::ConstNodeRef& value, std::string_view serial);
 static bool LoadTrackHashes();
 
-static std::array<const char*, static_cast<u32>(GameDatabase::Trait::Count)> s_trait_names = {{
-  "ForceInterpreter",
-  "ForceSoftwareRenderer",
-  "ForceSoftwareRendererForReadbacks",
-  "ForceInterlacing",
-  "DisableTrueColor",
-  "DisableUpscaling",
-  "DisableTextureFiltering",
-  "DisableScaledDithering",
-  "DisableForceNTSCTimings",
-  "DisableWidescreen",
-  "DisablePGXP",
-  "DisablePGXPCulling",
-  "DisablePGXPTextureCorrection",
-  "DisablePGXPColorCorrection",
-  "DisablePGXPDepthBuffer",
-  "ForcePGXPVertexCache",
-  "ForcePGXPCPUMode",
-  "ForceRecompilerMemoryExceptions",
-  "ForceRecompilerICache",
-  "ForceRecompilerLUTFastmem",
-  "IsLibCryptProtected",
+static constexpr const std::array<const char*, static_cast<int>(CompatibilityRating::Count)>
+  s_compatibility_rating_names = {
+    {"Unknown", "DoesntBoot", "CrashesInIntro", "CrashesInGame", "GraphicalAudioIssues", "NoIssues"}};
+
+static constexpr const std::array<const char*, static_cast<size_t>(CompatibilityRating::Count)>
+  s_compatibility_rating_display_names = {{TRANSLATE_NOOP("GameListCompatibilityRating", "Unknown"),
+                                           TRANSLATE_NOOP("GameListCompatibilityRating", "Doesn't Boot"),
+                                           TRANSLATE_NOOP("GameListCompatibilityRating", "Crashes In Intro"),
+                                           TRANSLATE_NOOP("GameListCompatibilityRating", "Crashes In-Game"),
+                                           TRANSLATE_NOOP("GameListCompatibilityRating", "Graphical/Audio Issues"),
+                                           TRANSLATE_NOOP("GameListCompatibilityRating", "No Issues")}};
+
+static constexpr const std::array<const char*, static_cast<u32>(GameDatabase::Trait::Count)> s_trait_names = {{
+  "forceInterpreter",
+  "forceSoftwareRenderer",
+  "forceSoftwareRendererForReadbacks",
+  "forceInterlacing",
+  "disableTrueColor",
+  "disableUpscaling",
+  "disableTextureFiltering",
+  "disableScaledDithering",
+  "disableForceNTSCTimings",
+  "disableWidescreen",
+  "disablePGXP",
+  "disablePGXPCulling",
+  "disablePGXPTextureCorrection",
+  "disablePGXPColorCorrection",
+  "disablePGXPDepthBuffer",
+  "forcePGXPVertexCache",
+  "forcePGXPCPUMode",
+  "forceRecompilerMemoryExceptions",
+  "forceRecompilerICache",
+  "forceRecompilerLUTFastmem",
+  "isLibCryptProtected",
 }};
 
+static constexpr const char* GAMEDB_YAML_FILENAME = "gamedb.yaml";
+static constexpr const char* DISCDB_YAML_FILENAME = "discdb.yaml";
+
 static bool s_loaded = false;
 static bool s_track_hashes_loaded = false;
 
@@ -84,6 +96,94 @@ static PreferUnorderedStringMap<u32> s_code_lookup;
 static TrackHashesMap s_track_hashes_map;
 } // namespace GameDatabase
 
+// RapidYAML utility routines.
+
+ALWAYS_INLINE std::string_view to_stringview(const c4::csubstr& s)
+{
+  return std::string_view(s.data(), s.size());
+}
+
+ALWAYS_INLINE std::string_view to_stringview(const c4::substr& s)
+{
+  return std::string_view(s.data(), s.size());
+}
+
+ALWAYS_INLINE c4::csubstr to_csubstr(const std::string_view& sv)
+{
+  return c4::csubstr(sv.data(), sv.length());
+}
+
+static bool GetStringFromObject(const ryml::ConstNodeRef& object, std::string_view key, std::string* dest)
+{
+  dest->clear();
+
+  const ryml::ConstNodeRef member = object.find_child(to_csubstr(key));
+  if (!member.valid())
+    return false;
+
+  const c4::csubstr val = member.val();
+  if (!val.empty())
+    dest->assign(val.data(), val.size());
+
+  return true;
+}
+
+template<typename T>
+static bool GetUIntFromObject(const ryml::ConstNodeRef& object, std::string_view key, T* dest)
+{
+  *dest = 0;
+
+  const ryml::ConstNodeRef member = object.find_child(to_csubstr(key));
+  if (!member.valid())
+    return false;
+
+  const c4::csubstr val = member.val();
+  if (val.empty())
+  {
+    Log_ErrorFmt("Unexpected empty value in {}", key);
+    return false;
+  }
+
+  const std::optional<T> opt_value = StringUtil::FromChars<T>(to_stringview(val));
+  if (!opt_value.has_value())
+  {
+    Log_ErrorFmt("Unexpected non-uint value in {}", key);
+    return false;
+  }
+
+  *dest = opt_value.value();
+  return true;
+}
+
+template<typename T>
+static std::optional<T> GetOptionalTFromObject(const ryml::ConstNodeRef& object, std::string_view key)
+{
+  std::optional<T> ret;
+
+  const ryml::ConstNodeRef member = object.find_child(to_csubstr(key));
+  if (member.valid())
+  {
+    const c4::csubstr val = member.val();
+    if (!val.empty())
+    {
+      ret = StringUtil::FromChars<T>(to_stringview(val));
+      if (!ret.has_value())
+      {
+        if constexpr (std::is_floating_point_v<T>)
+          Log_ErrorFmt("Unexpected non-float value in {}", key);
+        else if constexpr (std::is_integral_v<T>)
+          Log_ErrorFmt("Unexpected non-int value in {}", key);
+      }
+    }
+    else
+    {
+      Log_ErrorFmt("Unexpected empty value in {}", key);
+    }
+  }
+
+  return ret;
+}
+
 void GameDatabase::EnsureLoaded()
 {
   if (s_loaded)
@@ -98,11 +198,11 @@ void GameDatabase::EnsureLoaded()
     s_entries = {};
     s_code_lookup = {};
 
-    LoadGameDBJson();
+    LoadGameDBYaml();
     SaveToCache();
   }
 
-  Log_InfoPrintf("Database load took %.2f ms", timer.GetTimeMilliseconds());
+  Log_InfoFmt("Database load took {:.0f}ms", timer.GetTimeMilliseconds());
 }
 
 void GameDatabase::Unload()
@@ -198,30 +298,16 @@ GameDatabase::Entry* GameDatabase::GetMutableEntry(const std::string_view& seria
   return nullptr;
 }
 
-const char* GameDatabase::GetTraitName(Trait trait)
-{
-  DebugAssert(trait < Trait::Count);
-  return s_trait_names[static_cast<u32>(trait)];
-}
-
 const char* GameDatabase::GetCompatibilityRatingName(CompatibilityRating rating)
 {
-  static std::array<const char*, static_cast<int>(CompatibilityRating::Count)> names = {
-    {"Unknown", "DoesntBoot", "CrashesInIntro", "CrashesInGame", "GraphicalAudioIssues", "NoIssues"}};
-  return names[static_cast<int>(rating)];
+  return s_compatibility_rating_names[static_cast<int>(rating)];
 }
 
 const char* GameDatabase::GetCompatibilityRatingDisplayName(CompatibilityRating rating)
 {
-  static constexpr std::array<const char*, static_cast<size_t>(CompatibilityRating::Count)> names = {
-    {TRANSLATE_NOOP("GameListCompatibilityRating", "Unknown"),
-     TRANSLATE_NOOP("GameListCompatibilityRating", "Doesn't Boot"),
-     TRANSLATE_NOOP("GameListCompatibilityRating", "Crashes In Intro"),
-     TRANSLATE_NOOP("GameListCompatibilityRating", "Crashes In-Game"),
-     TRANSLATE_NOOP("GameListCompatibilityRating", "Graphical/Audio Issues"),
-     TRANSLATE_NOOP("GameListCompatibilityRating", "No Issues")}};
   return (rating >= CompatibilityRating::Unknown && rating < CompatibilityRating::Count) ?
-           Host::TranslateToCString("GameListCompatibilityRating", names[static_cast<int>(rating)]) :
+           Host::TranslateToCString("GameListCompatibilityRating",
+                                    s_compatibility_rating_display_names[static_cast<int>(rating)]) :
            "";
 }
 
@@ -580,7 +666,7 @@ bool GameDatabase::LoadFromCache()
     return false;
   }
 
-  const u64 gamedb_ts = Host::GetResourceFileTimestamp("gamedb.json", false).value_or(0);
+  const u64 gamedb_ts = Host::GetResourceFileTimestamp("gamedb.yaml", false).value_or(0);
 
   u32 signature, version, num_entries, num_codes;
   u64 file_gamedb_ts;
@@ -674,7 +760,7 @@ bool GameDatabase::LoadFromCache()
 
 bool GameDatabase::SaveToCache()
 {
-  const u64 gamedb_ts = Host::GetResourceFileTimestamp("gamedb.json", false).value_or(0);
+  const u64 gamedb_ts = Host::GetResourceFileTimestamp("gamedb.yaml", false).value_or(0);
 
   std::unique_ptr<ByteStream> stream(
     ByteStream::OpenFile(GetCacheFile().c_str(), BYTESTREAM_OPEN_CREATE | BYTESTREAM_OPEN_WRITE |
@@ -742,329 +828,258 @@ bool GameDatabase::SaveToCache()
   return true;
 }
 
-//////////////////////////////////////////////////////////////////////////
-// JSON Parsing
-//////////////////////////////////////////////////////////////////////////
-
-static bool GetStringFromObject(const rapidjson::Value& object, const char* key, std::string* dest)
+void GameDatabase::SetRymlCallbacks()
 {
-  dest->clear();
-  auto member = object.FindMember(key);
-  if (member == object.MemberEnd() || !member->value.IsString())
-    return false;
-
-  dest->assign(member->value.GetString(), member->value.GetStringLength());
-  return true;
+  ryml::Callbacks callbacks = ryml::get_callbacks();
+  callbacks.m_error = [](const char* msg, size_t msg_len, ryml::Location loc, void* userdata) {
+    Log_ErrorFmt("Parse error at {}:{} (bufpos={}): {}", loc.line, loc.col, loc.offset, std::string_view(msg, msg_len));
+  };
+  ryml::set_callbacks(callbacks);
+  c4::set_error_callback(
+    [](const char* msg, size_t msg_size) { Log_ErrorFmt("C4 error: {}", std::string_view(msg, msg_size)); });
 }
 
-static bool GetBoolFromObject(const rapidjson::Value& object, const char* key, bool* dest)
+bool GameDatabase::LoadGameDBYaml()
 {
-  *dest = false;
+  Common::Timer timer;
 
-  auto member = object.FindMember(key);
-  if (member == object.MemberEnd() || !member->value.IsBool())
-    return false;
-
-  *dest = member->value.GetBool();
-  return true;
-}
-
-template<typename T>
-static bool GetUIntFromObject(const rapidjson::Value& object, const char* key, T* dest)
-{
-  *dest = 0;
-
-  auto member = object.FindMember(key);
-  if (member == object.MemberEnd() || !member->value.IsUint())
-    return false;
-
-  *dest = static_cast<T>(member->value.GetUint());
-  return true;
-}
-
-static bool GetArrayOfStringsFromObject(const rapidjson::Value& object, const char* key, std::vector<std::string>* dest)
-{
-  dest->clear();
-  auto member = object.FindMember(key);
-  if (member == object.MemberEnd() || !member->value.IsArray())
-    return false;
-
-  for (const rapidjson::Value& str : member->value.GetArray())
-  {
-    if (str.IsString())
-    {
-      dest->emplace_back(str.GetString(), str.GetStringLength());
-    }
-  }
-  return true;
-}
-
-template<typename T>
-static std::optional<T> GetOptionalIntFromObject(const rapidjson::Value& object, const char* key)
-{
-  auto member = object.FindMember(key);
-  if (member == object.MemberEnd() || !member->value.IsInt())
-    return std::nullopt;
-
-  return static_cast<T>(member->value.GetInt());
-}
-
-template<typename T>
-static std::optional<T> GetOptionalUIntFromObject(const rapidjson::Value& object, const char* key)
-{
-  auto member = object.FindMember(key);
-  if (member == object.MemberEnd() || !member->value.IsUint())
-    return std::nullopt;
-
-  return static_cast<T>(member->value.GetUint());
-}
-
-static std::optional<float> GetOptionalFloatFromObject(const rapidjson::Value& object, const char* key)
-{
-  auto member = object.FindMember(key);
-  if (member == object.MemberEnd() || !member->value.IsFloat())
-    return std::nullopt;
-
-  return member->value.GetFloat();
-}
-
-bool GameDatabase::LoadGameDBJson()
-{
-  std::optional<std::string> gamedb_data(Host::ReadResourceFileToString("gamedb.json", false));
+  const std::optional<std::string> gamedb_data = Host::ReadResourceFileToString(GAMEDB_YAML_FILENAME, false);
   if (!gamedb_data.has_value())
   {
-    Log_ErrorPrintf("Failed to read game database");
+    Log_ErrorPrint("Failed to read game database");
     return false;
   }
 
-  // TODO: Parse in-place, avoid string allocations.
-  std::unique_ptr<rapidjson::Document> json = std::make_unique<rapidjson::Document>();
-  json->Parse(gamedb_data->c_str(), gamedb_data->size());
-  if (json->HasParseError())
-  {
-    Log_ErrorPrintf("Failed to parse game database: %s at offset %zu",
-                    rapidjson::GetParseError_En(json->GetParseError()), json->GetErrorOffset());
-    return false;
-  }
+  SetRymlCallbacks();
 
-  if (!json->IsArray())
-  {
-    Log_ErrorPrintf("Document is not an array");
-    return false;
-  }
+  const ryml::Tree tree = ryml::parse_in_arena(to_csubstr(GAMEDB_YAML_FILENAME), to_csubstr(gamedb_data.value()));
+  const ryml::ConstNodeRef root = tree.rootref();
+  s_entries.reserve(root.num_children());
 
-  const auto& jarray = json->GetArray();
-  s_entries.reserve(jarray.Size());
-
-  for (const rapidjson::Value& current : json->GetArray())
+  for (const ryml::ConstNodeRef& current : root.children())
   {
     // TODO: binary sort
     const u32 index = static_cast<u32>(s_entries.size());
     Entry& entry = s_entries.emplace_back();
-    if (!ParseJsonEntry(&entry, current))
+    if (!ParseYamlEntry(&entry, current))
     {
       s_entries.pop_back();
       continue;
     }
 
-    ParseJsonCodes(index, current);
+    ParseYamlCodes(index, current, entry.serial);
   }
 
-  Log_InfoPrintf("Loaded %zu entries and %zu codes from database", s_entries.size(), s_code_lookup.size());
-  return true;
+  ryml::reset_callbacks();
+
+  Log_InfoFmt("Loaded {} entries and {} codes from database in {:.0f}ms.", s_entries.size(), s_code_lookup.size(),
+              timer.GetTimeMilliseconds());
+  return !s_entries.empty();
 }
 
-bool GameDatabase::ParseJsonEntry(Entry* entry, const rapidjson::Value& value)
+bool GameDatabase::ParseYamlEntry(Entry* entry, const ryml::ConstNodeRef& value)
 {
-  if (!value.IsObject())
+  entry->serial = to_stringview(value.key());
+  if (entry->serial.empty())
   {
-    Log_WarningPrintf("entry is not an object");
+    Log_ErrorPrint("Missing serial for entry.");
     return false;
   }
 
-  if (!GetStringFromObject(value, "serial", &entry->serial) || !GetStringFromObject(value, "name", &entry->title) ||
-      entry->serial.empty())
+  GetStringFromObject(value, "name", &entry->title);
+
+  if (const ryml::ConstNodeRef metadata = value.find_child(to_csubstr("metadata")); metadata.valid())
   {
-    Log_ErrorPrintf("Missing serial or title for entry");
-    return false;
-  }
+    GetStringFromObject(metadata, "genre", &entry->genre);
+    GetStringFromObject(metadata, "developer", &entry->developer);
+    GetStringFromObject(metadata, "publisher", &entry->publisher);
 
-  GetStringFromObject(value, "genre", &entry->genre);
-  GetStringFromObject(value, "developer", &entry->developer);
-  GetStringFromObject(value, "publisher", &entry->publisher);
+    GetUIntFromObject(metadata, "minPlayers", &entry->min_players);
+    GetUIntFromObject(metadata, "maxPlayers", &entry->max_players);
+    GetUIntFromObject(metadata, "minBlocks", &entry->min_blocks);
+    GetUIntFromObject(metadata, "maxBlocks", &entry->max_blocks);
 
-  GetUIntFromObject(value, "minPlayers", &entry->min_players);
-  GetUIntFromObject(value, "maxPlayers", &entry->max_players);
-  GetUIntFromObject(value, "minBlocks", &entry->min_blocks);
-  GetUIntFromObject(value, "maxBlocks", &entry->max_blocks);
-
-  entry->release_date = 0;
-  {
-    std::string release_date;
-    if (GetStringFromObject(value, "releaseDate", &release_date))
+    entry->release_date = 0;
     {
-      std::istringstream iss(release_date);
-      struct tm parsed_time = {};
-      iss >> std::get_time(&parsed_time, "%Y-%m-%d");
-      if (!iss.fail())
+      std::string release_date;
+      if (GetStringFromObject(metadata, "releaseDate", &release_date))
       {
-        parsed_time.tm_isdst = 0;
+        std::istringstream iss(release_date);
+        struct tm parsed_time = {};
+        iss >> std::get_time(&parsed_time, "%Y-%m-%d");
+        if (!iss.fail())
+        {
+          parsed_time.tm_isdst = 0;
 #ifdef _WIN32
-        entry->release_date = _mkgmtime(&parsed_time);
+          entry->release_date = _mkgmtime(&parsed_time);
 #else
-        entry->release_date = timegm(&parsed_time);
+          entry->release_date = timegm(&parsed_time);
 #endif
+        }
       }
     }
   }
 
   entry->supported_controllers = static_cast<u16>(~0u);
-  const auto controllers = value.FindMember("controllers");
-  if (controllers != value.MemberEnd())
-  {
-    if (controllers->value.IsArray())
-    {
-      bool first = true;
-      for (const rapidjson::Value& controller : controllers->value.GetArray())
-      {
-        if (!controller.IsString())
-        {
-          Log_WarningPrintf("controller is not a string");
-          return false;
-        }
 
-        std::optional<ControllerType> ctype = Settings::ParseControllerTypeName(controller.GetString());
-        if (!ctype.has_value())
+  if (const ryml::ConstNodeRef controllers = value.find_child(to_csubstr("controllers"));
+      controllers.valid() && controllers.has_children())
+  {
+    bool first = true;
+    for (const ryml::ConstNodeRef& controller : controllers.children())
+    {
+      const std::string_view controller_str = to_stringview(controller.val());
+      if (controller_str.empty())
+      {
+        Log_WarningFmt("controller is not a string in {}", entry->serial);
+        return false;
+      }
+
+      std::optional<ControllerType> ctype = Settings::ParseControllerTypeName(controller_str);
+      if (!ctype.has_value())
+      {
+        Log_WarningFmt("Invalid controller type {} in {}", controller_str, entry->serial);
+        continue;
+      }
+
+      if (first)
+      {
+        entry->supported_controllers = 0;
+        first = false;
+      }
+
+      entry->supported_controllers |= (1u << static_cast<u16>(ctype.value()));
+    }
+  }
+
+  if (const ryml::ConstNodeRef compatibility = value.find_child(to_csubstr("compatibility"));
+      compatibility.valid() && compatibility.has_children())
+  {
+    const ryml::ConstNodeRef rating = compatibility.find_child(to_csubstr("rating"));
+    if (rating.valid())
+    {
+      const std::string_view rating_str = to_stringview(rating.val());
+
+      const auto iter = std::find(s_compatibility_rating_names.begin(), s_compatibility_rating_names.end(), rating_str);
+      if (iter != s_compatibility_rating_names.end())
+      {
+        const size_t rating_idx = static_cast<size_t>(std::distance(s_compatibility_rating_names.begin(), iter));
+        DebugAssert(rating_idx < static_cast<size_t>(CompatibilityRating::Count));
+        entry->compatibility = static_cast<CompatibilityRating>(rating_idx);
+      }
+      else
+      {
+        Log_WarningFmt("Unknown compatibility rating {} in {}", rating_str, entry->serial);
+      }
+    }
+  }
+
+  if (const ryml::ConstNodeRef traits = value.find_child(to_csubstr("traits")); traits.valid() && traits.has_children())
+  {
+    for (const ryml::ConstNodeRef& trait : traits.children())
+    {
+      const std::string_view trait_str = to_stringview(trait.val());
+      if (trait_str.empty())
+      {
+        Log_WarningFmt("Empty trait in {}", entry->serial);
+        continue;
+      }
+
+      const auto iter = std::find(s_trait_names.begin(), s_trait_names.end(), trait_str);
+      if (iter == s_trait_names.end())
+      {
+        Log_WarningFmt("Unknown trait {} in {}", trait_str, entry->serial);
+        continue;
+      }
+
+      const size_t trait_idx = static_cast<size_t>(std::distance(s_trait_names.begin(), iter));
+      DebugAssert(trait_idx < static_cast<size_t>(Trait::Count));
+      entry->traits[trait_idx] = true;
+    }
+  }
+
+  if (const ryml::ConstNodeRef settings = value.find_child(to_csubstr("settings"));
+      settings.valid() && settings.has_children())
+  {
+    entry->display_active_start_offset = GetOptionalTFromObject<s16>(settings, "displayActiveStartOffset");
+    entry->display_active_end_offset = GetOptionalTFromObject<s16>(settings, "displayActiveEndOffset");
+    entry->display_line_start_offset = GetOptionalTFromObject<s8>(settings, "displayLineStartOffset");
+    entry->display_line_end_offset = GetOptionalTFromObject<s8>(settings, "displayLineEndOffset");
+    entry->dma_max_slice_ticks = GetOptionalTFromObject<u32>(settings, "dmaMaxSliceTicks");
+    entry->dma_halt_ticks = GetOptionalTFromObject<u32>(settings, "dmaHaltTicks");
+    entry->gpu_fifo_size = GetOptionalTFromObject<u32>(settings, "gpuFIFOSize");
+    entry->gpu_max_run_ahead = GetOptionalTFromObject<u32>(settings, "gpuMaxRunAhead");
+    entry->gpu_pgxp_tolerance = GetOptionalTFromObject<float>(settings, "gpuPGXPTolerance");
+    entry->gpu_pgxp_depth_threshold = GetOptionalTFromObject<float>(settings, "gpuPGXPDepthThreshold");
+  }
+
+  if (const ryml::ConstNodeRef disc_set = value.find_child("discSet"); disc_set.valid() && disc_set.has_children())
+  {
+    GetStringFromObject(disc_set, "name", &entry->disc_set_name);
+
+    if (const ryml::ConstNodeRef set_serials = disc_set.find_child("serials");
+        set_serials.valid() && set_serials.has_children())
+    {
+      entry->disc_set_serials.reserve(set_serials.num_children());
+      for (const ryml::ConstNodeRef& serial : set_serials)
+      {
+        const std::string_view serial_str = to_stringview(serial.val());
+        if (serial_str.empty())
         {
-          Log_WarningPrintf("Invalid controller type '%s'", controller.GetString());
+          Log_WarningFmt("Empty disc set serial in {}", entry->serial);
           continue;
         }
 
-        if (first)
+        if (std::find(entry->disc_set_serials.begin(), entry->disc_set_serials.end(), serial_str) !=
+            entry->disc_set_serials.end())
         {
-          entry->supported_controllers = 0;
-          first = false;
+          Log_WarningFmt("Duplicate serial {} in disc set serials for {}", serial_str, entry->serial);
+          continue;
         }
 
-        entry->supported_controllers |= (1u << static_cast<u16>(ctype.value()));
+        entry->disc_set_serials.emplace_back(serial_str);
       }
     }
-    else
-    {
-      Log_WarningPrintf("controllers is not an array");
-    }
-  }
-
-  const auto compatibility = value.FindMember("compatibility");
-  if (compatibility != value.MemberEnd())
-  {
-    if (compatibility->value.IsObject())
-    {
-      u32 rating;
-      if (GetUIntFromObject(compatibility->value, "rating", &rating) &&
-          rating < static_cast<u32>(CompatibilityRating::Count))
-      {
-        entry->compatibility = static_cast<CompatibilityRating>(rating);
-      }
-    }
-    else
-    {
-      Log_WarningPrintf("compatibility is not an object");
-    }
-  }
-
-  const auto traits = value.FindMember("traits");
-  if (traits != value.MemberEnd())
-  {
-    if (traits->value.IsObject())
-    {
-      const auto& traitsobj = traits->value;
-      for (u32 trait = 0; trait < static_cast<u32>(Trait::Count); trait++)
-      {
-        bool bvalue;
-        if (GetBoolFromObject(traitsobj, s_trait_names[trait], &bvalue) && bvalue)
-          entry->traits[trait] = bvalue;
-      }
-
-      entry->display_active_start_offset = GetOptionalIntFromObject<s16>(traitsobj, "DisplayActiveStartOffset");
-      entry->display_active_end_offset = GetOptionalIntFromObject<s16>(traitsobj, "DisplayActiveEndOffset");
-      entry->display_line_start_offset = GetOptionalIntFromObject<s8>(traitsobj, "DisplayLineStartOffset");
-      entry->display_line_end_offset = GetOptionalIntFromObject<s8>(traitsobj, "DisplayLineEndOffset");
-      entry->dma_max_slice_ticks = GetOptionalUIntFromObject<u32>(traitsobj, "DMAMaxSliceTicks");
-      entry->dma_halt_ticks = GetOptionalUIntFromObject<u32>(traitsobj, "DMAHaltTicks");
-      entry->gpu_fifo_size = GetOptionalUIntFromObject<u32>(traitsobj, "GPUFIFOSize");
-      entry->gpu_max_run_ahead = GetOptionalUIntFromObject<u32>(traitsobj, "GPUMaxRunAhead");
-      entry->gpu_pgxp_tolerance = GetOptionalFloatFromObject(traitsobj, "GPUPGXPTolerance");
-      entry->gpu_pgxp_depth_threshold = GetOptionalFloatFromObject(traitsobj, "GPUPGXPDepthThreshold");
-    }
-    else
-    {
-      Log_WarningPrintf("traits is not an object");
-    }
-  }
-
-  GetStringFromObject(value, "discSetName", &entry->disc_set_name);
-  const auto disc_set_serials = value.FindMember("discSetSerials");
-  if (disc_set_serials != value.MemberEnd())
-  {
-    if (disc_set_serials->value.IsArray())
-    {
-      const auto disc_set_serials_array = disc_set_serials->value.GetArray();
-      entry->disc_set_serials.reserve(disc_set_serials_array.Size());
-      for (const rapidjson::Value& serial : disc_set_serials_array)
-      {
-        if (serial.IsString())
-        {
-          entry->disc_set_serials.emplace_back(serial.GetString(), serial.GetStringLength());
-        }
-        else
-        {
-          Log_WarningPrintf("discSetSerial is not a string");
-        }
-      }
-    }
-    else
-    {
-      Log_WarningPrintf("discSetSerials is not an array");
-    }
   }
 
   return true;
 }
 
-bool GameDatabase::ParseJsonCodes(u32 index, const rapidjson::Value& value)
+bool GameDatabase::ParseYamlCodes(u32 index, const ryml::ConstNodeRef& value, std::string_view serial)
 {
-  auto member = value.FindMember("codes");
-  if (member == value.MemberEnd())
+  const ryml::ConstNodeRef& codes = value.find_child(to_csubstr("codes"));
+  if (!codes.valid() || !codes.has_children())
   {
-    Log_WarningPrintf("codes member is missing");
-    return false;
-  }
+    // use serial instead
+    auto iter = s_code_lookup.find(serial);
+    if (iter != s_code_lookup.end())
+    {
+      Log_WarningFmt("Duplicate code '{}'", serial);
+      return false;
+    }
 
-  if (!member->value.IsArray())
-  {
-    Log_WarningPrintf("codes is not an array");
-    return false;
+    s_code_lookup.emplace(serial, index);
+    return true;
   }
 
   u32 added = 0;
-  for (const rapidjson::Value& current_code : member->value.GetArray())
+  for (const ryml::ConstNodeRef& current_code : codes)
   {
-    if (!current_code.IsString())
+    const std::string_view current_code_str = to_stringview(current_code.val());
+    if (current_code_str.empty())
     {
-      Log_WarningPrintf("code is not a string");
+      Log_WarningFmt("code is not a string in {}", serial);
       continue;
     }
 
-    const std::string_view code(current_code.GetString(), current_code.GetStringLength());
-    auto iter = s_code_lookup.find(code);
+    auto iter = s_code_lookup.find(current_code_str);
     if (iter != s_code_lookup.end())
     {
-      Log_WarningPrintf("Duplicate code '%.*s'", static_cast<int>(code.size()), code.data());
+      Log_WarningFmt("Duplicate code '{}' in {}", current_code_str, serial);
       continue;
     }
 
-    s_code_lookup.emplace(code, index);
+    s_code_lookup.emplace(current_code_str, index);
     added++;
   }
 
@@ -1082,105 +1097,84 @@ void GameDatabase::EnsureTrackHashesMapLoaded()
 
 bool GameDatabase::LoadTrackHashes()
 {
-  std::optional<std::string> gamedb_data(Host::ReadResourceFileToString("gamedb.json", false));
+  Common::Timer load_timer;
+
+  std::optional<std::string> gamedb_data(Host::ReadResourceFileToString(DISCDB_YAML_FILENAME, false));
   if (!gamedb_data.has_value())
   {
-    Log_ErrorPrintf("Failed to read game database");
+    Log_ErrorPrint("Failed to read game database");
     return false;
   }
 
+  SetRymlCallbacks();
+
   // TODO: Parse in-place, avoid string allocations.
-  std::unique_ptr<rapidjson::Document> json = std::make_unique<rapidjson::Document>();
-  json->Parse(gamedb_data->c_str(), gamedb_data->size());
-  if (json->HasParseError())
-  {
-    Log_ErrorPrintf("Failed to parse game database: %s at offset %zu",
-                    rapidjson::GetParseError_En(json->GetParseError()), json->GetErrorOffset());
-    return false;
-  }
-
-  if (!json->IsArray())
-  {
-    Log_ErrorPrintf("Document is not an array");
-    return false;
-  }
+  const ryml::Tree tree = ryml::parse_in_arena(to_csubstr(DISCDB_YAML_FILENAME), to_csubstr(gamedb_data.value()));
+  const ryml::ConstNodeRef root = tree.rootref();
 
   s_track_hashes_map = {};
 
-  for (const rapidjson::Value& current : json->GetArray())
+  size_t serials = 0;
+  for (const ryml::ConstNodeRef& current : root.children())
   {
-    if (!current.IsObject())
+    const std::string_view serial = to_stringview(current.key());
+    if (serial.empty() || !current.has_children())
     {
-      Log_WarningPrintf("entry is not an object");
+      Log_WarningPrint("entry is not an object");
       continue;
     }
 
-    std::vector<std::string> codes;
-    if (!GetArrayOfStringsFromObject(current, "codes", &codes))
+    const ryml::ConstNodeRef track_data = current.find_child(to_csubstr("trackData"));
+    if (!track_data.valid() || !track_data.has_children())
     {
-      Log_WarningPrintf("codes member is missing");
+      Log_WarningFmt("trackData is missing in {}", serial);
       continue;
     }
 
-    auto track_data = current.FindMember("track_data");
-    if (track_data == current.MemberEnd())
+    u32 revision = 0;
+    for (const ryml::ConstNodeRef& track_revisions : track_data.children())
     {
-      Log_WarningPrintf("track_data member is missing");
-      continue;
-    }
-
-    if (!track_data->value.IsArray())
-    {
-      Log_WarningPrintf("track_data is not an array");
-      continue;
-    }
-
-    uint32_t revision = 0;
-    for (const rapidjson::Value& track_revisions : track_data->value.GetArray())
-    {
-      if (!track_revisions.IsObject())
+      const ryml::ConstNodeRef tracks = track_revisions.find_child(to_csubstr("tracks"));
+      if (!tracks.valid() || !tracks.has_children())
       {
-        Log_WarningPrintf("track_data is not an array of object");
+        Log_WarningFmt("tracks member is missing in {}", serial);
         continue;
       }
 
-      auto tracks = track_revisions.FindMember("tracks");
-      if (tracks == track_revisions.MemberEnd())
-      {
-        Log_WarningPrintf("tracks member is missing");
-        continue;
-      }
+      std::string revision_string;
+      GetStringFromObject(track_revisions, "version", &revision_string);
 
-      if (!tracks->value.IsArray())
+      for (const ryml::ConstNodeRef& track : tracks)
       {
-        Log_WarningPrintf("tracks is not an array");
-        continue;
-      }
-
-      std::string revisionString;
-      GetStringFromObject(track_revisions, "version", &revisionString);
-
-      for (const rapidjson::Value& track : tracks->value.GetArray())
-      {
-        auto md5_field = track.FindMember("md5");
-        if (md5_field == track.MemberEnd() || !md5_field->value.IsString())
+        const ryml::ConstNodeRef md5 = track.find_child("md5");
+        std::string_view md5_str;
+        if (!md5.valid() || (md5_str = to_stringview(md5.val())).empty())
         {
+          Log_WarningFmt("md5 is missing in track in {}", serial);
           continue;
         }
 
-        auto md5 = CDImageHasher::HashFromString(
-          std::string_view(md5_field->value.GetString(), md5_field->value.GetStringLength()));
-        if (md5)
+        const std::optional<CDImageHasher::Hash> md5o = CDImageHasher::HashFromString(md5_str);
+        if (md5o.has_value())
         {
-          s_track_hashes_map.emplace(std::piecewise_construct, std::forward_as_tuple(md5.value()),
-                                     std::forward_as_tuple(codes, revisionString, revision));
+          s_track_hashes_map.emplace(std::piecewise_construct, std::forward_as_tuple(md5o.value()),
+                                     std::forward_as_tuple(std::string(serial), revision_string, revision));
+        }
+        else
+        {
+          Log_WarningFmt("invalid md5 in {}", serial);
         }
       }
       revision++;
     }
+
+    serials++;
   }
 
-  return true;
+  ryml::reset_callbacks();
+  Log_InfoFmt("Loaded {} track hashes from {} serials in {:.0f}ms.", s_track_hashes_map.size(), serials,
+              load_timer.GetTimeMilliseconds());
+  return !s_track_hashes_map.empty();
 }
 
 const GameDatabase::TrackHashesMap& GameDatabase::GetTrackHashesMap()

src/core/game_database.h

@@ -1,4 +1,4 @@
-// SPDX-FileCopyrightText: 2019-2022 Connor McLaughlin <stenzek@gmail.com>
+// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
 // SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
 
 #pragma once
@@ -98,16 +98,14 @@ const Entry* GetEntryForSerial(const std::string_view& serial);
 std::string GetSerialForDisc(CDImage* image);
 std::string GetSerialForPath(const char* path);
 
-const char* GetTraitName(Trait trait);
-
 const char* GetCompatibilityRatingName(CompatibilityRating rating);
 const char* GetCompatibilityRatingDisplayName(CompatibilityRating rating);
 
 /// Map of track hashes for image verification
 struct TrackData
 {
-  TrackData(std::vector<std::string> codes, std::string revisionString, uint32_t revision)
-    : codes(std::move(codes)), revisionString(revisionString), revision(revision)
+  TrackData(std::string serial_, std::string revision_str_, uint32_t revision_)
+    : serial(std::move(serial_)), revision_str(std::move(revision_str_)), revision(revision_)
   {
   }
 
@@ -115,12 +113,12 @@ struct TrackData
   {
     // 'revisionString' is deliberately ignored in comparisons as it's redundant with comparing 'revision'! Do not
     // change!
-    return left.codes == right.codes && left.revision == right.revision;
+    return left.serial == right.serial && left.revision == right.revision;
   }
 
-  std::vector<std::string> codes;
-  std::string revisionString;
-  uint32_t revision;
+  std::string serial;
+  std::string revision_str;
+  u32 revision;
 };
 
 using TrackHashesMap = std::multimap<CDImageHasher::Hash, TrackData>;

src/duckstation-qt/gamesummarywidget.cpp

@@ -1,4 +1,4 @@
-// SPDX-FileCopyrightText: 2019-2022 Connor McLaughlin <stenzek@gmail.com>
+// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
 // SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
 
 #include "gamesummarywidget.h"
@@ -13,7 +13,6 @@
 
 #include "fmt/format.h"
 
-#include <QtConcurrent/QtConcurrent>
 #include <QtCore/QFuture>
 #include <QtWidgets/QMessageBox>
 
@@ -219,13 +218,6 @@ void GameSummaryWidget::onComputeHashClicked()
     return;
   }
 
-#ifndef _DEBUGFAST
-  // Kick off hash preparation asynchronously, as building the map of results may take a while
-  // This breaks for DebugFast because of the iterator debug level mismatch.
-  QFuture<const GameDatabase::TrackHashesMap*> result =
-    QtConcurrent::run([]() { return &GameDatabase::GetTrackHashesMap(); });
-#endif
-
   QtModalProgressCallback progress_callback(this);
   progress_callback.SetProgressRange(image->GetTrackCount());
 
@@ -259,6 +251,7 @@ void GameSummaryWidget::onComputeHashClicked()
   if (calculate_hash_success)
   {
     std::string found_revision;
+    std::string found_serial;
     m_redump_search_keyword = CDImageHasher::HashToString(track_hashes.front());
 
     progress_callback.SetStatusText("Verifying hashes...");
@@ -270,11 +263,7 @@ void GameSummaryWidget::onComputeHashClicked()
     // 2. For each data track match, try to match all audio tracks
     //    If all match, assume this revision. Else, try other revisions,
     //    and accept the one with the most matches.
-#ifndef _DEBUGFAST
-    const GameDatabase::TrackHashesMap& hashes_map = *result.result();
-#else
     const GameDatabase::TrackHashesMap& hashes_map = GameDatabase::GetTrackHashesMap();
-#endif
 
     auto data_track_matches = hashes_map.equal_range(track_hashes[0]);
     if (data_track_matches.first != data_track_matches.second)
@@ -317,13 +306,30 @@ void GameSummaryWidget::onComputeHashClicked()
         }
       }
 
-      found_revision = best_data_match->second.revisionString;
+      found_revision = best_data_match->second.revision_str;
+      found_serial = best_data_match->second.serial;
     }
 
+    QString text;
+
     if (!found_revision.empty())
+      text = tr("Revision: %1").arg(found_revision.empty() ? tr("N/A") : QString::fromStdString(found_revision));
+
+    if (found_serial != m_ui.serial->text().toStdString())
     {
-      m_ui.revision->setText(
-        tr("Revision: %1").arg(found_revision.empty() ? tr("N/A") : QString::fromStdString(found_revision)));
+      text =
+        tr("Serial Mismatch: %1 vs %2%3").arg(QString::fromStdString(found_serial)).arg(m_ui.serial->text()).arg(text);
+    }
+
+    if (!text.isEmpty())
+    {
+      if (m_ui.verifySpacer)
+      {
+        m_ui.verifyLayout->removeItem(m_ui.verifySpacer);
+        delete m_ui.verifySpacer;
+        m_ui.verifySpacer = nullptr;
+      }
+      m_ui.revision->setText(text);
       m_ui.revision->setVisible(true);
     }
   }

src/duckstation-qt/gamesummarywidget.ui

@@ -189,9 +189,9 @@
     <widget class="QComboBox" name="inputProfile"/>
    </item>
    <item row="16" column="1">
-    <layout class="QHBoxLayout" name="horizontalLayout_2" stretch="1,0,0">
+    <layout class="QHBoxLayout" name="verifyLayout" stretch="0,1,0">
      <item>
-      <spacer name="horizontalSpacer">
+      <spacer name="verifySpacer">
        <property name="orientation">
         <enum>Qt::Horizontal</enum>
        </property>
@@ -207,7 +207,7 @@
       <widget class="QLineEdit" name="revision">
        <property name="minimumSize">
         <size>
-         <width>100</width>
+         <width>300</width>
          <height>0</height>
         </size>
        </property>