recompiler: more robust code cache allocation (#1242)

- align static code cache to 64 KiB (up from 4 KiB)
- remove dependency on alignas() and associated build workaround
- check for mprotect() failure and fall back on dynamic allocation
- perform dynamic allocation (if needed) at top of main on all platforms

ares/ares/memory/fixed-allocator.cpp

@@ -1,21 +1,32 @@
 #include <ares/ares.hpp>
 
+#if !defined(PLATFORM_MACOS)
+#define STATIC_ALLOCATION
+#endif
+
 namespace ares::Memory {
 
 constexpr u32 fixedBufferSize = 128_MiB;
 
-#if defined(PLATFORM_MACOS)
-//dynamic allocation for unsupported platforms
-FixedAllocator::FixedAllocator() {
-  _allocator.resize(fixedBufferSize, bump_allocator::executable);
-}
-#else
-alignas(4096) u8 fixedBuffer[fixedBufferSize];
+#if defined(STATIC_ALLOCATION)
+u8 fixedBuffer[fixedBufferSize + 64_KiB];
+#endif
 
 FixedAllocator::FixedAllocator() {
-  _allocator.resize(sizeof(fixedBuffer), 0, fixedBuffer);
+  u8* buffer = nullptr;
+
+  #if defined(STATIC_ALLOCATION)
+  //align to 64 KiB (maximum page size of any supported OS)
+  auto offset = -(uintptr)fixedBuffer % 64_KiB;
+  //set protection to executable
+  if(memory::protect(fixedBuffer + offset, fixedBufferSize, true)) {
+    //use static allocation
+    buffer = fixedBuffer + offset;
+  }
+  #endif
+
+  _allocator.resize(fixedBufferSize, bump_allocator::executable, buffer);
 }
-#endif
 
 auto FixedAllocator::get() -> bump_allocator& {
   static FixedAllocator allocator;

desktop-ui/desktop-ui.cpp

@@ -44,10 +44,8 @@ auto locate(const string& name) -> string {
 
 #include <nall/main.hpp>
 auto nall::main(Arguments arguments) -> void {
-#if defined(PLATFORM_MACOS)
   //force early allocation for better proximity to executable code
   ares::Memory::FixedAllocator::get();
-#endif
 
 #if defined(PLATFORM_WINDOWS)
   bool createTerminal = arguments.take("--terminal");

nall/GNUmakefile

@@ -246,10 +246,6 @@ ifeq ($(findstring clang++,$(compiler)),clang++)
   ifneq ($(platform),macos)
     options += -fuse-ld=lld
   endif
-  ifeq ($(arch),arm64)
-    # work around bad interaction with alignas(n) when n >= 4096
-    flags += -mno-global-merge
-  endif
 # gcc settings
 else ifeq ($(findstring g++,$(compiler)),g++)
   flags += -fno-strict-aliasing -fwrapv -Wno-trigraphs

nall/bump-allocator.hpp

@@ -28,9 +28,6 @@ struct bump_allocator {
     reset();
 
     if(buffer) {
-      if(flags & executable) {
-        memory::protect(buffer, capacity, true);
-      }
       if(flags & zero_fill) {
         memset(buffer, 0x00, capacity);
       }

nall/memory.cpp

@@ -29,18 +29,17 @@ NALL_HEADER_INLINE auto unmap(void* target, u32 size) -> void {
   #endif
 }
 
-NALL_HEADER_INLINE auto protect(void* target, u32 size, bool executable) -> void {
+NALL_HEADER_INLINE auto protect(void* target, u32 size, bool executable) -> bool {
   #if defined(API_WINDOWS)
   DWORD protect = executable ? PAGE_EXECUTE_READWRITE : PAGE_READWRITE;
   DWORD oldProtect;
-  VirtualProtect(target, size, protect, &oldProtect);
+  return VirtualProtect(target, size, protect, &oldProtect);
   #elif defined(API_POSIX)
   int prot = PROT_READ | PROT_WRITE;
   if(executable) {
     prot |= PROT_EXEC;
   }
-  int ret = mprotect(target, size, prot);
-  assert(ret == 0);
+  return !mprotect(target, size, prot);
   #endif
 }
 

nall/memory.hpp

@@ -34,7 +34,7 @@ namespace nall::memory {
 
   auto map(u32 size, bool executable) -> void*;
   auto unmap(void* target, u32 size) -> void;
-  auto protect(void* target, u32 size, bool executable) -> void;
+  auto protect(void* target, u32 size, bool executable) -> bool;
   auto jitprotect(bool executable) -> void;
 }
 
@@ -195,12 +195,6 @@ template<u32 size, typename T> auto writem(void* target, T data) -> void {
   for(s32 n = size - 1; n >= 0; n--) *p++ = data >> n * 8;
 }
 
-auto map(u32 size, bool executable) -> void*;
-
-auto unmap(void* target, u32 size) -> void;
-
-auto protect(void* target, u32 size, bool executable) -> void;
-
 inline auto jitprotect(bool executable) -> void {
   #if defined(PLATFORM_MACOS)
   if(__builtin_available(macOS 11.0, *)) {