Core Info Cache Improvements

- Core info cache can now be enabled/disabled on all platforms via a new `Settings > Core > Cache Core Info Files` option
- Core info cache file has been renamed from `.cache` to `core_info.cache` (i.e. it is no longer a 'hidden' file on Unix platforms, so can be deleted easily)
- The core info cache file is now compressed (rzip) to further reduce disk IO
- The presence of a `core_info.refresh` file in the core info directory will force a one-time refresh of the info cache. This file is generated automatically when toggling on the `Cache Core Info Files` option, and we will also add it to core info file packaging such that updating info files (either manually or via the online updater) will force a refresh
- The core info cache no longer contains 'core is locked' and 'firmware missing' data fields; these are 'dynamic' properties that must be determined at runtime
- The 'core is locked' status is now determined on core info intialisation by parsing the core directory listing, rather than by performing indivdual 'lock file exists' checks. This minimises file IO, and greatly improves performance on devices with slow storage
- While parsing the core info cache file, we now avoid unnecessary `strdup()`s when adding entries to the resultant cache list
- Memory leaks (potential and real) have been fixed, and safety checks added
- Build errors have been fixed

config.def.h

@@ -243,6 +243,15 @@
 #endif
 #define DEFAULT_CHECK_FIRMWARE_BEFORE_LOADING false
 
+/* Specifies whether to cache core info
+ * into a single (compressed) file for improved
+ * load times on platforms with slow IO */
+#if defined(_3DS)
+#define DEFAULT_CORE_INFO_CACHE_ENABLE true
+#else
+#define DEFAULT_CORE_INFO_CACHE_ENABLE false
+#endif
+
 /* Specifies whether to 'reload' (fork and quit)
  * RetroArch when launching content with the
  * currently loaded core

configuration.c

@@ -1485,6 +1485,7 @@ static struct config_bool_setting *populate_settings_bool(
    SETTING_BOOL("input_descriptor_hide_unbound", &settings->bools.input_descriptor_hide_unbound, true, input_descriptor_hide_unbound, false);
    SETTING_BOOL("load_dummy_on_core_shutdown",   &settings->bools.load_dummy_on_core_shutdown, true, DEFAULT_LOAD_DUMMY_ON_CORE_SHUTDOWN, false);
    SETTING_BOOL("check_firmware_before_loading", &settings->bools.check_firmware_before_loading, true, DEFAULT_CHECK_FIRMWARE_BEFORE_LOADING, false);
+   SETTING_BOOL("core_info_cache_enable",        &settings->bools.core_info_cache_enable, true, DEFAULT_CORE_INFO_CACHE_ENABLE, false);
 #ifndef HAVE_DYNAMIC
    SETTING_BOOL("always_reload_core_on_run_content", &settings->bools.always_reload_core_on_run_content, true, DEFAULT_ALWAYS_RELOAD_CORE_ON_RUN_CONTENT, false);
 #endif

configuration.h

@@ -777,6 +777,7 @@ typedef struct settings
       bool network_remote_enable_user[MAX_USERS];
       bool load_dummy_on_core_shutdown;
       bool check_firmware_before_loading;
+      bool core_info_cache_enable;
 #ifndef HAVE_DYNAMIC
       bool always_reload_core_on_run_content;
 #endif

core_info.h

@@ -110,14 +110,6 @@ typedef struct
    size_t info_count;
 } core_info_list_t;
 
-typedef struct 
-{
-   core_info_t *items;
-   size_t length;
-   size_t capacity;
-   bool refresh;
-} core_info_cache_list_t;
-
 typedef struct core_info_ctx_firmware
 {
    const char *path;
@@ -165,7 +157,7 @@ bool core_info_get_current_core(core_info_t **core);
 void core_info_deinit_list(void);
 
 bool core_info_init_list(const char *path_info, const char *dir_cores,
-      const char *exts, bool show_hidden_files);
+      const char *exts, bool show_hidden_files, bool enable_cache);
 
 bool core_info_get_list(core_info_list_t **core);
 
@@ -212,13 +204,11 @@ core_info_state_t *coreinfo_get_ptr(void);
 
 bool core_info_core_file_id_is_equal(const char *core_path_a, const char *core_path_b);
 
-core_info_t *core_info_get_cache(core_info_cache_list_t *list, char *core_file_id);
-void core_info_add_cache(core_info_cache_list_t *list, core_info_t *info);
-void core_info_copy(core_info_t *src, core_info_t *dst);
-void core_info_write_cache_file(core_info_cache_list_t *list, const char *info_dir);
-core_info_cache_list_t *core_info_read_cache_file(const char *info_dir);
-core_info_cache_list_t *new_core_info_cache_list(void);
-void core_info_check_uninstalled(core_info_cache_list_t *list);
+/* When called, generates a temporary file
+ * that will force an info cache refresh the
+ * next time that core info is initialised with
+ * caching enabled */
+bool core_info_cache_force_refresh(const char *path_info);
 
 RETRO_END_DECLS
 

file_path_special.h

@@ -106,10 +106,13 @@ RETRO_BEGIN_DECLS
 #define FILE_PATH_CORE_BACKUP_EXTENSION ".lcbk"
 #define FILE_PATH_CORE_BACKUP_EXTENSION_NO_DOT "lcbk"
 #define FILE_PATH_LOCK_EXTENSION ".lck"
+#define FILE_PATH_LOCK_EXTENSION_NO_DOT "lck"
 #define FILE_PATH_BACKUP_EXTENSION ".bak"
 #if defined(RARCH_MOBILE)
 #define FILE_PATH_DEFAULT_OVERLAY "gamepads/neo-retropad/neo-retropad.cfg"
 #endif
+#define FILE_PATH_CORE_INFO_CACHE "core_info.cache"
+#define FILE_PATH_CORE_INFO_CACHE_REFRESH "core_info.refresh"
 
 enum application_special_type
 {

intl/msg_hash_lbl.h

@@ -1172,6 +1172,10 @@ MSG_HASH(
    MENU_ENUM_LABEL_CHECK_FOR_MISSING_FIRMWARE,
    "check_for_missing_firmware"
    )
+MSG_HASH(
+   MENU_ENUM_LABEL_CORE_INFO_CACHE_ENABLE,
+   "core_info_cache_enable"
+   )
 MSG_HASH(
    MENU_ENUM_LABEL_DUMMY_ON_CORE_SHUTDOWN,
    "dummy_on_core_shutdown"

intl/msg_hash_us.h

@@ -2886,6 +2886,14 @@ MSG_HASH(
    MENU_ENUM_SUBLABEL_CHECK_FOR_MISSING_FIRMWARE,
    "Check if all the required firmware is present before attempting to load content."
    )
+MSG_HASH(
+   MENU_ENUM_LABEL_VALUE_CORE_INFO_CACHE_ENABLE,
+   "Cache Core Info Files"
+   )
+MSG_HASH(
+   MENU_ENUM_SUBLABEL_CORE_INFO_CACHE_ENABLE,
+   "Maintain a persistent local cache of installed core information. Greatly reduces loading times on platforms with slow disk access."
+   )
 #ifndef HAVE_DYNAMIC
 MSG_HASH(
    MENU_ENUM_LABEL_VALUE_ALWAYS_RELOAD_CORE_ON_RUN_CONTENT,

menu/cbs/menu_cbs_sublabel.c

@@ -357,6 +357,7 @@ DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_video_adaptive_vsync,          MENU_
 DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_core_allow_rotate,             MENU_ENUM_SUBLABEL_VIDEO_ALLOW_ROTATE)
 DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_dummy_on_core_shutdown,        MENU_ENUM_SUBLABEL_DUMMY_ON_CORE_SHUTDOWN)
 DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_dummy_check_missing_firmware,  MENU_ENUM_SUBLABEL_CHECK_FOR_MISSING_FIRMWARE)
+DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_core_info_cache_enable,        MENU_ENUM_SUBLABEL_CORE_INFO_CACHE_ENABLE)
 #ifndef HAVE_DYNAMIC
 DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_always_reload_core_on_run_content, MENU_ENUM_SUBLABEL_ALWAYS_RELOAD_CORE_ON_RUN_CONTENT)
 #endif
@@ -3403,6 +3404,9 @@ int menu_cbs_init_bind_sublabel(menu_file_list_cbs_t *cbs,
          case MENU_ENUM_LABEL_CHECK_FOR_MISSING_FIRMWARE:
             BIND_ACTION_SUBLABEL(cbs, action_bind_sublabel_dummy_check_missing_firmware);
             break;
+         case MENU_ENUM_LABEL_CORE_INFO_CACHE_ENABLE:
+            BIND_ACTION_SUBLABEL(cbs, action_bind_sublabel_core_info_cache_enable);
+            break;
 #ifndef HAVE_DYNAMIC
          case MENU_ENUM_LABEL_ALWAYS_RELOAD_CORE_ON_RUN_CONTENT:
             BIND_ACTION_SUBLABEL(cbs, action_bind_sublabel_always_reload_core_on_run_content);

menu/menu_displaylist.c

@@ -8618,11 +8618,12 @@ unsigned menu_displaylist_build_list(
       case DISPLAYLIST_CORE_SETTINGS_LIST:
          {
             menu_displaylist_build_info_t build_list[] = {
-               {MENU_ENUM_LABEL_VIDEO_SHARED_CONTEXT,  PARSE_ONLY_BOOL},
-               {MENU_ENUM_LABEL_DRIVER_SWITCH_ENABLE,  PARSE_ONLY_BOOL},
-               {MENU_ENUM_LABEL_DUMMY_ON_CORE_SHUTDOWN, PARSE_ONLY_BOOL},
-               {MENU_ENUM_LABEL_CHECK_FOR_MISSING_FIRMWARE, PARSE_ONLY_BOOL},
-               {MENU_ENUM_LABEL_VIDEO_ALLOW_ROTATE,    PARSE_ONLY_BOOL},
+               {MENU_ENUM_LABEL_VIDEO_SHARED_CONTEXT,              PARSE_ONLY_BOOL},
+               {MENU_ENUM_LABEL_DRIVER_SWITCH_ENABLE,              PARSE_ONLY_BOOL},
+               {MENU_ENUM_LABEL_DUMMY_ON_CORE_SHUTDOWN,            PARSE_ONLY_BOOL},
+               {MENU_ENUM_LABEL_CHECK_FOR_MISSING_FIRMWARE,        PARSE_ONLY_BOOL},
+               {MENU_ENUM_LABEL_VIDEO_ALLOW_ROTATE,                PARSE_ONLY_BOOL},
+               {MENU_ENUM_LABEL_CORE_INFO_CACHE_ENABLE,            PARSE_ONLY_BOOL},
 #ifndef HAVE_DYNAMIC
                {MENU_ENUM_LABEL_ALWAYS_RELOAD_CORE_ON_RUN_CONTENT, PARSE_ONLY_BOOL},
 #endif

menu/menu_setting.c

@@ -7809,6 +7809,20 @@ static void general_write_handler(rarch_setting_t *setting)
             task_push_wifi_disable(NULL);
 #endif
          break;
+      case MENU_ENUM_LABEL_CORE_INFO_CACHE_ENABLE:
+         {
+            settings_t *settings           = config_get_ptr();
+            const char *dir_libretro       = settings->paths.directory_libretro;
+            const char *path_libretro_info = settings->paths.path_libretro_info;
+
+            /* When enabling the core info cache,
+             * force a cache refresh on the next
+             * core info initialisation */
+            if (*setting->value.target.boolean)
+               core_info_cache_force_refresh(!string_is_empty(path_libretro_info) ?
+                     path_libretro_info : dir_libretro);
+         }
+         break;
       default:
          break;
    }
@@ -9327,9 +9341,9 @@ static bool setting_append_list(
          {
             unsigned i, listing = 0;
 #ifndef HAVE_DYNAMIC
-            struct bool_entry bool_entries[7];
+            struct bool_entry bool_entries[8];
 #else
-            struct bool_entry bool_entries[6];
+            struct bool_entry bool_entries[7];
 #endif
             START_GROUP(list, list_info, &group_info,
                   msg_hash_to_str(MENU_ENUM_LABEL_VALUE_CORE_SETTINGS), parent_group);
@@ -9384,6 +9398,13 @@ static bool setting_append_list(
             bool_entries[listing].flags          = SD_FLAG_ADVANCED;
             listing++;
 
+            bool_entries[listing].target         = &settings->bools.core_info_cache_enable;
+            bool_entries[listing].name_enum_idx  = MENU_ENUM_LABEL_CORE_INFO_CACHE_ENABLE;
+            bool_entries[listing].SHORT_enum_idx = MENU_ENUM_LABEL_VALUE_CORE_INFO_CACHE_ENABLE;
+            bool_entries[listing].default_value  = DEFAULT_CORE_INFO_CACHE_ENABLE;
+            bool_entries[listing].flags          = SD_FLAG_NONE;
+            listing++;
+
 #ifndef HAVE_DYNAMIC
             bool_entries[listing].target         = &settings->bools.always_reload_core_on_run_content;
             bool_entries[listing].name_enum_idx  = MENU_ENUM_LABEL_ALWAYS_RELOAD_CORE_ON_RUN_CONTENT;

msg_hash.h

@@ -2121,6 +2121,7 @@ enum msg_hash_enums
 
    MENU_LABEL(DUMMY_ON_CORE_SHUTDOWN),
    MENU_LABEL(CHECK_FOR_MISSING_FIRMWARE),
+   MENU_LABEL(CORE_INFO_CACHE_ENABLE),
 #ifndef HAVE_DYNAMIC
    MENU_LABEL(ALWAYS_RELOAD_CORE_ON_RUN_CONTENT),
 #endif

retroarch.c

@@ -14131,6 +14131,7 @@ bool command_event(enum event_command cmd, void *data)
             const char *dir_libretro       = settings->paths.directory_libretro;
             const char *path_libretro_info = settings->paths.path_libretro_info;
             bool show_hidden_files         = settings->bools.show_hidden_files;
+            bool core_info_cache_enable    = settings->bools.core_info_cache_enable;
 
             ext_name[0]                    = '\0';
 
@@ -14143,7 +14144,8 @@ bool command_event(enum event_command cmd, void *data)
                core_info_init_list(path_libretro_info,
                      dir_libretro,
                      ext_name,
-                     show_hidden_files
+                     show_hidden_files,
+                     core_info_cache_enable
                      );
          }
          break;

samples/tasks/database/main.c

@@ -66,7 +66,7 @@ int main(int argc, char *argv[])
 #else
    task_queue_init(false /* threaded enable */, main_msg_queue_push);
 #endif
-   core_info_init_list(core_info_dir, core_dir, exts, true);
+   core_info_init_list(core_info_dir, core_dir, exts, true, false);
 
    task_push_dbscan(playlist_dir, db_dir, input_dir, true,
          true, main_db_cb);