Back to the original topic, I was finally able to try the MMU protection method. Here's my example program:
#include <proto/exec.h>
#include <stdio.h>
#include <stdlib.h>
#define BUFFER_SIZE 48
#define MEM_ALIGN_SIZE 4096
#define MEM_GUARD_SIZE 4096
void *AllocMemory(int size, int protect)
{
UBYTE *ptr = IExec->AllocVecTags(MEM_GUARD_SIZE + size,
AVT_Type, MEMF_PRIVATE,
AVT_Alignment, MEM_ALIGN_SIZE,
TAG_END);
if (!ptr) return NULL;
if (protect)
{
struct MMUIFace *IMMU = (struct MMUIFace *)IExec->GetInterface((struct Library *)SysBase, (CONST_STRPTR)"MMU", 1, NULL);
ULONG attrs = IMMU->GetMemoryAttrs(ptr, 0);
IMMU->SetMemoryAttrs(ptr, MEM_GUARD_SIZE, (attrs | MEMATTRF_READ_ONLY));
IExec->DropInterface((struct Interface *)IMMU);
}
return ptr ? (ptr + MEM_GUARD_SIZE) : NULL;
}
void FreeMemory(void *block)
{
if (!block) return;
UBYTE *ptr = (UBYTE *)block;
IExec->FreeVec(ptr - MEM_GUARD_SIZE);
}
int main(int argc, char *argv[])
{
BOOL protect = (argc > 1) ? atoi(argv[1]) : 0;
printf("allocating %d bytes protect %d\n", BUFFER_SIZE, protect);
UBYTE *buf = AllocMemory(BUFFER_SIZE, protect);
if (!buf)
{
printf("allocation failed\n");
return 1;
}
printf("valid writes...\n");
for (int i = 0; i < BUFFER_SIZE; i++)
{
buf[i] = 0xFF;
}
printf("invalid write...\n");
buf[-1] = 0xFF;
printf("invalid writes...\n");
for (int i = 0; i < MEM_GUARD_SIZE; i++)
{
buf[-i] = 0xFF;
}
printf("freeing memory\n");
FreeMemory(buf);
return 0;
}
This works in my simple test case, but there's one catch: the actual project does a metric ton of small memory allocations. The 4KB overhead plus the alignment requirement makes the memory fragmented to the point where the allocations start to fail, even with a 512MB Z3 RAM expansion. I tried smaller guard blocks while keeping the alignment as 4KB, but that ended up causing all sorts of freezes and crashes.
I guess my only option is Linux? :(