From 509d85b2a3e440946d0b0c114a4a2ba9fc2dfe07 Mon Sep 17 00:00:00 2001 From: Sour Date: Tue, 12 Jun 2018 00:49:33 -0400 Subject: [PATCH] Linux: Makefile tweaks + support for PGO --- .gitignore | 6 ++- PGOHelper/PGOGames/readme.md | 7 +++ PGOHelper/PGOHelper.cpp | 84 ++++++++++++++++++++---------------- build.sh | 12 ++---- buildPGO.sh | 62 ++++++++++++++++++++++++++ makefile | 37 ++++++++++++++-- 6 files changed, 157 insertions(+), 51 deletions(-) create mode 100644 PGOHelper/PGOGames/readme.md create mode 100644 buildPGO.sh diff --git a/.gitignore b/.gitignore index 28a1d099..2690d7c4 100644 --- a/.gitignore +++ b/.gitignore @@ -171,4 +171,8 @@ $RECYCLE.BIN/ *.VC.db-shm Docs/docs/ -Docs/*.exe \ No newline at end of file +Docs/*.exe + +PGOHelper/PGOMesenHome +*.profraw +*.profdata \ No newline at end of file diff --git a/PGOHelper/PGOGames/readme.md b/PGOHelper/PGOGames/readme.md new file mode 100644 index 00000000..4bfd0574 --- /dev/null +++ b/PGOHelper/PGOGames/readme.md @@ -0,0 +1,7 @@ +This folder is used when compiling Mesen with PGO. +All .nes files put in this folder will be folder will be run in console mode for a few seconds as a way of building of a profile for use with PGO. + +Once you have added a few roms to this folder, run "make pgo" to produce a PGO-optimized binary (it will take several minutes to build) + +Another folder, called "PGOMesenHome" will be created alongside this one when the instrumented executable runs when executing the PGO script. +This folder will be used as a temporary home folder location for Mesen to store the files it creates in the process. \ No newline at end of file diff --git a/PGOHelper/PGOHelper.cpp b/PGOHelper/PGOHelper.cpp index 437eeaa8..ba0a5da4 100644 --- a/PGOHelper/PGOHelper.cpp +++ b/PGOHelper/PGOHelper.cpp @@ -1,7 +1,19 @@ +#ifdef _WIN32 +#else + #define __stdcall +#endif + #include #include #include #include +#include +#include +#include +namespace fs = std::experimental::filesystem; +using std::string; +using std::vector; +using std::thread; enum class VideoFilterType { @@ -37,67 +49,63 @@ enum class VideoFilterType extern "C" { void __stdcall SetFlags(uint64_t flags); void __stdcall SetVideoFilter(VideoFilterType filter); - void __stdcall InitializeEmu(char* homeFolder, void*, void*, bool, bool, bool); - void __stdcall LoadROM(const char* filename, char* patchFile); + void __stdcall InitializeEmu(const char* homeFolder, void*, void*, bool, bool, bool); + void __stdcall LoadROM(const char* filename, const char* patchFile); void __stdcall Run(); + void __stdcall Release(); void __stdcall Stop(); void __stdcall DebugInitialize(); } +vector GetFilesInFolder(string rootFolder, std::unordered_set extensions) +{ + vector files; + vector folders = { { rootFolder } }; + + std::error_code errorCode; + if(!fs::is_directory(fs::u8path(rootFolder), errorCode)) { + return files; + } + + for(string folder : folders) { + for(fs::directory_iterator i(fs::u8path(folder.c_str())), end; i != end; i++) { + string extension = i->path().extension().u8string(); + std::transform(extension.begin(), extension.end(), extension.begin(), ::tolower); + if(extensions.find(extension) != extensions.end()) { + files.push_back(i->path().u8string()); + } + } + } + + return files; +} + int main(int argc, char* argv[]) { - using namespace std; - vector testRoms{ - "..\\..\\Games\\Super Dodge Ball (USA).nes", - "..\\..\\Games\\Super Mario Bros. (USA).nes", - "..\\..\\Games\\Mega Man (USA).nes", - "..\\..\\Games\\Mega Man 2 (USA).nes", - "..\\..\\Games\\Mega Man 3 (USA).nes", - "..\\..\\Games\\Mega Man 4 (USA).nes", - "..\\..\\Games\\Mega Man 5 (USA).nes", - "..\\..\\Games\\Mega Man 6 (USA).nes", - "..\\..\\Games\\MMC5\\Just Breed (J) [!].nes", - "..\\..\\Games\\MMC5\\Castlevania III - Dracula's Curse (U) [!].nes", - "..\\..\\Games\\Blades of Steel (USA).nes", - "..\\..\\Games\\Kirby's Adventure (USA).nes", - "..\\..\\Games\\Legend of Zelda, The (USA).nes", - "..\\..\\Games\\Super Mario Bros. 3 (USA).nes", - "..\\..\\Games\\Teenage Mutant Ninja Turtles II - The Arcade Game (USA).nes", - "..\\..\\Games\\Dragon Warrior III (USA).nes", - "..\\..\\Games\\Dragon Warrior IV (USA).nes" - }; + vector testRoms = GetFilesInFolder("../PGOGames", { ".nes" }); + + string homeFolder = "../PGOMesenHome"; SetFlags(0x8000000000000000 | 0x20); //EmulationFlags::ConsoleMode | UseHdPacks - InitializeEmu("C:\\Code\\PGOMesen", nullptr, nullptr, false, false, false); - LoadROM(testRoms[0], ""); + InitializeEmu(homeFolder.c_str(), nullptr, nullptr, false, false, false); + LoadROM(testRoms[0].c_str(), ""); std::cout << "Running: " << testRoms[0] << std::endl; thread testThread([testRoms] { VideoFilterType filterTypes[13] = { - VideoFilterType::BisqwitNtscQuarterRes, - VideoFilterType::HQ2x, - VideoFilterType::HQ3x, - VideoFilterType::HQ4x, - VideoFilterType::NTSC, - VideoFilterType::Scale2x, - VideoFilterType::Scale3x, - VideoFilterType::Scale4x, - VideoFilterType::xBRZ2x, - VideoFilterType::xBRZ3x, - VideoFilterType::xBRZ4x, - VideoFilterType::xBRZ5x, - VideoFilterType::xBRZ6x, + VideoFilterType::BisqwitNtscQuarterRes, VideoFilterType::HQ2x, VideoFilterType::HQ3x, VideoFilterType::HQ4x, VideoFilterType::NTSC, VideoFilterType::Scale2x, VideoFilterType::Scale3x, VideoFilterType::Scale4x, VideoFilterType::xBRZ2x, VideoFilterType::xBRZ3x, VideoFilterType::xBRZ4x, VideoFilterType::xBRZ5x, VideoFilterType::xBRZ6x }; for(size_t i = 1; i < testRoms.size(); i++) { std::this_thread::sleep_for(std::chrono::duration(5000)); std::cout << "Running: " << testRoms[i] << std::endl; SetVideoFilter(filterTypes[i % 13]); - LoadROM(testRoms[i], ""); + LoadROM(testRoms[i].c_str(), ""); DebugInitialize(); } std::this_thread::sleep_for(std::chrono::duration(5000)); Stop(); + Release(); }); Run(); testThread.join(); diff --git a/build.sh b/build.sh index ac1ed102..ef22dcce 100755 --- a/build.sh +++ b/build.sh @@ -10,12 +10,8 @@ if [ "$1" = libretro ]; then MESENPLATFORM=x86 make clean LTO=true MESENPLATFORM=x86 make libretro -j 16 else - MESENPLATFORM=x64 make clean - LTO=true MESENPLATFORM=x64 make core -j 16 - - MESENPLATFORM=x86 make clean - LTO=true MESENPLATFORM=x86 make core -j 16 - - cp ./InteropDLL/obj.x64/libMesenCore.x64.dll ./bin - cp ./InteropDLL/obj.x86/libMesenCore.x86.dll ./bin + MESENPLATFORM=x64 BUILDTARGET=core ./buildPGO.sh + MESENPLATFORM=x86 BUILDTARGET=core ./buildPGO.sh + cp ./InteropDLL/obj.x64/libMesenCore.x64.dll ./bin/Any\ CPU/PGO\ Profile/Dependencies + cp ./InteropDLL/obj.x86/libMesenCore.x86.dll ./bin/Any\ CPU/PGO\ Profile/Dependencies fi diff --git a/buildPGO.sh b/buildPGO.sh new file mode 100644 index 00000000..726c3938 --- /dev/null +++ b/buildPGO.sh @@ -0,0 +1,62 @@ +#!/bin/sh + +# This build is used to compile an instrumented version of the binary, run it, and then optimize the binary with the profiling data. +# LTO is also enabled by default by this script. +# On clang, this results in a +60% speed boost compared to using just "make" (w/o LTO) +# +# Rom files must be copied to the PGOHelper/PGOGames folder beforehand - all *.nes files in that folder will be executed as part of the profiling process. +# Using a variety of roms is recommended (e.g different mappers, etc.) +# +# You can run this script via make: +# For clang, run "make pgo" +# For GCC, run "USE_GCC=true make pgo" +# +# Note: While GCC runs through this script just fine, the runtime performance is pretty terrible (something must be wrong with the way this is built) +# +# This will produce the following binary: bin/x64/Release/Mesen.exe + +if [ "$BUILDTARGET" = core ]; then + TARG="core" +else + TARG="" +fi + +if [ "$MESENPLATFORM" = x86 ]; then + PLAT="x86" +else + PLAT="x64" +fi + +OBJ="PGOHelper/obj.${PLAT}/" +FLAGS="LTO=true MESENPLATFORM=${PLAT}" + +eval ${FLAGS} make clean + +#create instrumented binary +eval ${FLAGS} PGO=profile make pgohelper -j 16 +eval cp InteropDLL/obj.${PLAT}/libMesenCore.${PLAT}.dll ${OBJ} + +#run the instrumented binary +cd ${OBJ} +./pgohelper +cd .. + +if [ "$USE_GCC" != true ]; then + #clang-specific steps to convert the profiling data and clean the files + llvm-profdata merge -output=pgo.profdata pgo.profraw + cd .. + eval ${FLAGS} make clean +else + cd .. +fi + +#rebuild using the profiling data to optimize +eval ${FLAGS} PGO=optimize make ${TARG} -j 16 -B + +if [ "$USE_GCC" != true ]; then + rm PGOHelper/pgo.profdata + rm PGOHelper/pgo.profraw +else + rm ./*.gcda +fi + diff --git a/makefile b/makefile index d0f25f14..1e231fc5 100644 --- a/makefile +++ b/makefile @@ -20,10 +20,19 @@ MESENFLAGS= libretro : MESENFLAGS=-D LIBRETRO -CPPC=clang++ -GCCOPTIONS=-fPIC -Wall --std=c++14 -O3 $(MESENFLAGS) -Wno-parentheses -Wno-switch +ifeq ($(USE_GCC),true) + CPPC=g++ + CC=gcc + PROFILE_GEN_FLAG=-fprofile-generate + PROFILE_USE_FLAG=-fprofile-use +else + CPPC=clang++ + CC=clang + PROFILE_GEN_FLAG = -fprofile-instr-generate=$(CURDIR)/PGOHelper/pgo.profraw + PROFILE_USE_FLAG = -fprofile-instr-use=$(CURDIR)/PGOHelper/pgo.profdata +endif -CC=clang +GCCOPTIONS=-fPIC -Wall --std=c++14 -O3 $(MESENFLAGS) -Wno-parentheses -Wno-switch CCOPTIONS=-fPIC -Wall -O3 $(MESENFLAGS) ifeq ($(MESENPLATFORM),x86) @@ -42,6 +51,16 @@ ifeq ($(LTO),true) GCCOPTIONS += -flto endif +ifeq ($(PGO),profile) + CCOPTIONS += ${PROFILE_GEN_FLAG} + GCCOPTIONS += ${PROFILE_GEN_FLAG} +endif + +ifeq ($(PGO),optimize) + CCOPTIONS += ${PROFILE_USE_FLAG} + GCCOPTIONS += ${PROFILE_USE_FLAG} +endif + OBJFOLDER=obj.$(MESENPLATFORM) SHAREDLIB=libMesenCore.$(MESENPLATFORM).dll LIBRETROLIB=mesen_libretro.$(MESENPLATFORM).so @@ -52,6 +71,7 @@ UTILOBJ=$(patsubst Utilities/%.cpp,Utilities/$(OBJFOLDER)/%.o,$(wildcard Utiliti LINUXOBJ=$(patsubst Linux/%.cpp,Linux/$(OBJFOLDER)/%.o,$(wildcard Linux/*.cpp)) SEVENZIPOBJ=$(patsubst SevenZip/%.c,SevenZip/$(OBJFOLDER)/%.o,$(wildcard SevenZip/*.c)) LUAOBJ=$(patsubst Lua/%.c,Lua/$(OBJFOLDER)/%.o,$(wildcard Lua/*.c)) + ifeq ($(SYSTEM_LIBEVDEV), true) LIBEVDEVLIB=$(shell pkg-config --libs libevdev) LIBEVDEVINC=$(shell pkg-config --cflags libevdev) @@ -91,6 +111,9 @@ testhelper: InteropDLL/$(OBJFOLDER)/$(SHAREDLIB) $(CPPC) $(GCCOPTIONS) -Wl,-z,defs -o testhelper TestHelper/*.cpp InteropDLL/ConsoleWrapper.cpp $(SEVENZIPOBJ) $(LUAOBJ) $(LINUXOBJ) $(LIBEVDEVOBJ) $(UTILOBJ) $(COREOBJ) -pthread $(FSLIB) $(SDL2LIB) $(LIBEVDEVLIB) mv testhelper TestHelper/$(OBJFOLDER) +pgohelper: InteropDLL/$(OBJFOLDER)/$(SHAREDLIB) + mkdir -p PGOHelper/$(OBJFOLDER) && cd PGOHelper/$(OBJFOLDER) && $(CPPC) $(GCCOPTIONS) -Wl,-z,defs -o pgohelper ../PGOHelper.cpp ../../InteropDLL/$(OBJFOLDER)/$(SHAREDLIB) -pthread $(FSLIB) $(SDL2LIB) $(LIBEVDEVLIB) + SevenZip/$(OBJFOLDER)/%.o: SevenZip/%.c mkdir -p SevenZip/$(OBJFOLDER) && cd SevenZip/$(OBJFOLDER) && $(CC) $(CCOPTIONS) -c $(patsubst SevenZip/%, ../%, $<) Lua/$(OBJFOLDER)/%.o: Lua/%.c @@ -117,12 +140,17 @@ InteropDLL/$(OBJFOLDER)/$(SHAREDLIB): $(SEVENZIPOBJ) $(LUAOBJ) $(UTILOBJ) $(CORE $(CPPC) $(GCCOPTIONS) -Wl,-z,defs -shared -o $(SHAREDLIB) InteropDLL/*.cpp $(SEVENZIPOBJ) $(LUAOBJ) $(LINUXOBJ) $(LIBEVDEVOBJ) $(UTILOBJ) $(COREOBJ) $(SDL2INC) -pthread $(FSLIB) $(SDL2LIB) $(LIBEVDEVLIB) mv $(SHAREDLIB) InteropDLL/$(OBJFOLDER) - Libretro/$(OBJFOLDER)/$(LIBRETROLIB): $(SEVENZIPOBJ) $(UTILOBJ) $(COREOBJ) $(LUAOBJ) Libretro/libretro.cpp mkdir -p Libretro/$(OBJFOLDER) $(CPPC) $(GCCOPTIONS) -Wl,-z,defs -shared -o $(LIBRETROLIB) Libretro/*.cpp $(SEVENZIPOBJ) $(UTILOBJ) $(COREOBJ) $(LUAOBJ) -pthread $(FSLIB) mv $(LIBRETROLIB) Libretro/$(OBJFOLDER) +pgo: + ./buildPGO.sh + +official: + ./build.sh + debug: MONO_LOG_LEVEL=debug mono $(RELEASEFOLDER)/Mesen.exe @@ -138,4 +166,5 @@ clean: rm -rf InteropDLL/$(OBJFOLDER) rm -rf Libretro/$(OBJFOLDER) rm -rf TestHelper/$(OBJFOLDER) + rm -rf PGOHelper/$(OBJFOLDER) rm -rf $(RELEASEFOLDER)