1 /* *****************************************************************************
2 * Copyright (c) 2007-2014 Alexis Naveros.
4 * Ecere Corporation has unlimited/unrestricted rights.
5 * *****************************************************************************/
9 * Global memory management header.
17 #define OFFSET(s, m) ((unsigned int)(uintptr_t) (&((s *) 0)->m))
27 #define MM_INLINE_LIST_FUNCTIONS
29 #define MM_ALLOC_CHECK
32 #define MM_DEBUG_GUARD_BYTES (32)
33 #define MM_DEBUG_MMAP (1)
34 /* Enabling this will lead to ever growing memory usage! Strictly for debugging. */
35 #define MM_DEBUG_MMAP_LINGERING (0)
47 #if defined(__linux__) || defined(__gnu_linux__) || defined(__linux) || defined(__linux)
50 #elif defined(__APPLE__)
53 #elif defined(__unix__) || defined(__unix) || defined(unix)
55 #elif defined(_WIN64) || defined(__WIN64__) || defined(WIN64)
58 #define MM_WINDOWS (1)
59 #elif defined(_WIN32) || defined(__WIN32__) || defined(WIN32)
61 #define MM_WINDOWS (1)
65 #define MM_MINGW32 (1)
66 #define MM_MINGW64 (1)
68 #define MM_MINGW32 (1)
74 #define MM_DEBUG_MMAP (0)
78 #define MM_CPU_COUNT_MAXIMUM (1024)
79 #define MM_NODE_COUNT_MAXIMUM (256)
82 #ifndef CPUCONF_CACHE_LINE_SIZE
83 #define CPUCONF_CACHE_LINE_SIZE 64
88 #define MM_CACHE_ALIGN __attribute__((aligned(CPUCONF_CACHE_LINE_SIZE)))
89 #define MM_RESTRICT __restrict
90 #define MM_NOINLINE __attribute__((noinline))
92 #define MM_CACHE_ALIGN
98 #if defined(__GNUC__) || defined(__INTEL_COMPILER)
99 #define MM_ALIGN16 __attribute__((aligned(16)))
100 #define MM_ALIGN16_SAFE (1)
101 #elif defined(_MSC_VER)
102 #define MM_ALIGN16 __declspec(align(16))
103 #define MM_ALIGN16_SAFE (1)
106 #define MM_ALIGN16_SAFE (0)
111 #define MM_ERROR() {printf("MM Error at %s:%d\n",file,line);exit(1)}
114 #define ADDRESS(p,o) ((void *)(((char *)p)+(o)))
118 #define ADDRESSDIFF(a,b) (((char *)a)-((char *)b))
122 #include <sys/time.h>
125 int mmGetTimeOfDay( struct timeval *tv );
126 #define gettimeofday(a,b) mmGetTimeOfDay(a)
136 int cpunode[MM_CPU_COUNT_MAXIMUM];
137 int64_t nodesize[MM_NODE_COUNT_MAXIMUM];
138 int nodecpucount[MM_NODE_COUNT_MAXIMUM];
142 extern mmContext mmcontext;
145 #include "mmatomic.h"
146 #include "mmthread.h"
150 #define MM_FUNC(n) mm##n##Debug
151 #define MM_PARAMS , const char *file, int line
153 #define MM_FUNC(n) mm##n
165 void mmThreadBindToNode( int nodeindex );
166 void mmThreadBindToNode( int nodeindex );
167 void mmThreadBindToCpu( int cpuindex );
168 int mmCpuGetNode( int cpuindex );
170 void *mmNodeAlloc( int nodeindex, size_t size );
171 void mmNodeFree( int nodeindex, void *v, size_t size );
172 void mmNodeMap( int nodeindex, void *start, size_t bytes );
174 void *mmNodeAlignAlloc( int nodeindex, size_t size, intptr_t align );
175 void mmNodeAlignFree( int nodeindex, void *v, size_t size );
201 #ifndef MM_INLINE_LIST_FUNCTIONS
203 void mmListAdd( void **list, void *item, intptr_t offset );
204 void mmListRemove( void *item, intptr_t offset );
205 void mmListMergeList( void **listdst, void **listsrc, intptr_t offset );
206 void mmListMoveList( void **listdst, void **listsrc, intptr_t offset );
208 void mmListDualInit( mmListDualHead *head );
209 void mmListDualAddFirst( mmListDualHead *head, void *item, intptr_t offset );
210 void mmListDualAddLast( mmListDualHead *head, void *item, intptr_t offset );
211 void mmListDualInsertAfter( mmListDualHead *head, void **prevnext, void *item, intptr_t offset );
212 void mmListDualRemove( mmListDualHead *head, void *item, intptr_t offset );
213 void *mmListDualLast( mmListDualHead *head, intptr_t offset );
214 void *mmListDualPrevious( mmListDualHead *head, void *item, intptr_t offset );
218 static inline void mmListAdd( void **list, void *item, intptr_t offset )
220 mmListNode *node, *next;
221 node = ADDRESS( item, offset );
226 next = ADDRESS( *list, offset );
227 next->prev = &(node->next);
233 static inline void mmListRemove( void *item, intptr_t offset )
235 mmListNode *node, *next;
236 node = ADDRESS( item, offset );
237 *(node->prev) = (void *)node->next;
240 next = ADDRESS( node->next, offset );
241 next->prev = node->prev;
246 static inline void mmListMergeList( void **listdst, void **listsrc, intptr_t offset )
252 for( item = *listdst ; item ; item = node->next )
254 node = ADDRESS( item, offset );
255 listdst = &node->next;
258 node = ADDRESS( item, offset );
259 node->prev = listdst;
265 static inline void mmListMoveList( void **listdst, void **listsrc, intptr_t offset )
275 node = ADDRESS( item, offset );
276 node->prev = listdst;
282 static inline void mmListDualInit( mmListDualHead *head )
285 head->last = &head->first;
289 static inline void mmListDualAddFirst( mmListDualHead *head, void *item, intptr_t offset )
291 mmListNode *node, *next;
292 node = ADDRESS( item, offset );
293 node->prev = &head->first;
294 node->next = head->first;
297 next = ADDRESS( node->next, offset );
298 next->prev = &(node->next);
301 head->last = &(node->next);
306 static inline void mmListDualAddLast( mmListDualHead *head, void *item, intptr_t offset )
312 node = ADDRESS( item, offset );
313 node->prev = head->last;
314 head->last = &(node->next);
319 static inline void mmListDualInsertAfter( mmListDualHead *head, void **prevnext, void *item, intptr_t offset )
321 mmListNode *node, *next;
322 node = ADDRESS( item, offset );
323 node->prev = prevnext;
324 node->next = *prevnext;
327 next = ADDRESS( *prevnext, offset );
328 next->prev = &(node->next);
331 head->last = &(node->next);
336 static inline void mmListDualRemove( mmListDualHead *head, void *item, intptr_t offset )
338 mmListNode *node, *next;
339 node = ADDRESS( item, offset );
340 *(node->prev) = (void *)node->next;
343 next = ADDRESS( node->next, offset );
344 next->prev = node->prev;
347 head->last = node->prev;
351 static inline void *mmListDualLast( mmListDualHead *head, intptr_t offset )
353 if( !( head->first ) )
355 return ADDRESS( head->last, -( offset + OFFSET(mmListNode,next) ) );
358 static inline void *mmListDualPrevious( mmListDualHead *head, void *item, intptr_t offset )
361 if( item == head->first )
363 node = ADDRESS( item, offset );
364 return ADDRESS( node->prev, -( offset + OFFSET(mmListNode,next) ) );
369 void mmListLoopInit( mmListLoopHead *head );
370 void mmListLoopAddFirst( mmListLoopHead *head, void *item, intptr_t offset );
371 void mmListLoopAddLast( mmListLoopHead *head, void *item, intptr_t offset );
372 void mmListLoopInsert( mmListLoopHead *head, void *previtem, void *item, intptr_t offset );
373 void mmListLoopRemove( mmListLoopHead *head, void *item, intptr_t offset );
374 void *mmListLoopLast( mmListLoopHead *head, intptr_t offset );
389 #define MM_BTREE_FLAGS_LEFT (0)
390 #define MM_BTREE_FLAGS_RIGHT (1)
391 #define MM_BTREE_FLAGS_DIRECTION_MASK (1)
392 #define MM_BTREE_FLAGS_STEP (2)
394 void mmBTreeInsert( void *item, void *parent, int itemflag, intptr_t offset, void **root );
395 void mmBTreeRemove( void *item, intptr_t offset, void **root );
397 void *mmBtreeMostLeft( void *root, intptr_t offset );
398 void *mmBtreeMostRight( void *root, intptr_t offset );
399 void *mmBtreeNeighbourLeft( void *item, intptr_t offset );
400 void *mmBtreeNeighbourRight( void *item, intptr_t offset );
401 intptr_t mmBtreeItemCount( void *root, intptr_t offset );
402 int mmBtreeListOrdered( void *root, intptr_t offset, int (*callback)( void *item, void *v ), void *v );
403 int mmBtreeListBalanced( void *root, intptr_t offset, int (*callback)( void *item, void *v ), void *v );
430 void *(*relayalloc)( void *head, size_t bytes MM_PARAMS );
431 void (*relayfree)( void *head, void *v, size_t bytes MM_PARAMS );
436 void MM_FUNC(BlockInit)( mmBlockHead *head, size_t chunksize, int chunkperblock, int keepfreecount, int alignment MM_PARAMS );
437 void MM_FUNC(BlockNodeInit)( mmBlockHead *head, int nodeindex, size_t chunksize, int chunkperblock, int keepfreecount, int alignment MM_PARAMS );
438 void *MM_FUNC(BlockAlloc)( mmBlockHead *head MM_PARAMS );
439 void MM_FUNC(BlockRelease)( mmBlockHead *head, void *v MM_PARAMS );
440 void MM_FUNC(BlockFree)( mmBlockHead *head, void *v MM_PARAMS );
441 void MM_FUNC(BlockFreeAll)( mmBlockHead *head MM_PARAMS );
442 void MM_FUNC(BlockProcessList)( mmBlockHead *head, void *userpointer, int (*processchunk)( void *chunk, void *userpointer ) MM_PARAMS );
443 int MM_FUNC(BlockUseCount)( mmBlockHead *head MM_PARAMS );
444 int MM_FUNC(BlockFreeCount)( mmBlockHead *head MM_PARAMS );
447 #define mmBlockInit(v,w,x,y,z) MM_FUNC(BlockInit)(v,w,x,y,z,__FILE__,__LINE__)
448 #define mmBlockNodeInit(u,v,w,x,y,z) MM_FUNC(BlockNodeInit)(u,v,w,x,y,z,__FILE__,__LINE__)
449 #define mmBlockAlloc(x) MM_FUNC(BlockAlloc)(x,__FILE__,__LINE__)
450 #define mmBlockRelease(x,y) MM_FUNC(BlockRelease)(x,y,__FILE__,__LINE__)
451 #define mmBlockFree(x,y) MM_FUNC(BlockFree)(x,y,__FILE__,__LINE__)
452 #define mmBlockFreeAll(x) MM_FUNC(BlockFreeAll)(x,__FILE__,__LINE__)
453 #define mmBlockProcessList(x,y,z) MM_FUNC(BlockProcessList)(x,y,z,__FILE__,__LINE__)
454 #define mmBlockUseCount(x) MM_FUNC(BlockProcessList)(x,__FILE__,__LINE__)
455 #define mmBlockFreeCount(x) MM_FUNC(BlockProcessList)(x,__FILE__,__LINE__)
459 void mmBlockRelayByVolume( mmBlockHead *head, void *volumehead );
460 void mmBlockRelayByZone( mmBlockHead *head, void *zonehead );
471 mmBlockHead indexblock;
477 void mmIndexInit( mmIndexHead *head, int indexesperblock );
478 void mmIndexFreeAll( mmIndexHead *head );
479 void mmIndexAdd( mmIndexHead *head, intptr_t index );
480 intptr_t mmIndexGet( mmIndexHead *head );
481 int mmIndexRemove( mmIndexHead *head, intptr_t index );
482 size_t mmIndexCount( mmIndexHead *head );
494 uintptr_t countalign;
495 uintptr_t indexshift;
503 void mmBitTableInit( mmBitTableHead *head, int bitsperentry, int chunksize, int initmask );
504 void mmBitTableFreeAll( mmBitTableHead *head );
505 void mmBitTableSet( mmBitTableHead *head, uintptr_t index, int flags, int editmask );
506 uintptr_t mmBitTableGet( mmBitTableHead *head, uintptr_t index );
528 int MM_FUNC(GrowInit)( mmGrow *mgrow, size_t nodesize MM_PARAMS );
529 void MM_FUNC(GrowFreeAll)( mmGrow *mgrow MM_PARAMS );
530 void *MM_FUNC(GrowAlloc)( mmGrow *mgrow, size_t bytes MM_PARAMS );
531 void MM_FUNC(GrowRewindLast)( mmGrow *mgrow, size_t rewind MM_PARAMS );
534 #define mmGrowInit(x,y) MM_FUNC(GrowInit)(x,y,__FILE__,__LINE__)
535 #define mmGrowFreeAll(x) MM_FUNC(GrowFreeAll)(x,__FILE__,__LINE__)
536 #define mmGrowAlloc(x,y) MM_FUNC(GrowAlloc)(x,y,__FILE__,__LINE__)
537 #define mmGrowRewindLast(x) MM_FUNC(GrowRewindLast)(x,__FILE__,__LINE__)
557 #define MM_DIR_ENTRY(dir,index) ( (dir)->table[ index >> (dir)->pageshift ][ index & (dir)->pagemask ] )
559 int MM_FUNC(DirInit)( mmDirectory *dir, intptr_t pageshift, intptr_t pagecount MM_PARAMS );
560 void MM_FUNC(DirSize)( mmDirectory *dir, intptr_t size MM_PARAMS );
561 void MM_FUNC(DirSet)( mmDirectory *dir, intptr_t index, void *entry MM_PARAMS );
562 void *MM_FUNC(DirGet)( mmDirectory *dir, intptr_t index MM_PARAMS );
563 void MM_FUNC(DirSetFast)( mmDirectory *dir, intptr_t index, void *entry MM_PARAMS );
564 void *MM_FUNC(DirGetFast)( mmDirectory *dir, intptr_t index MM_PARAMS );
565 void MM_FUNC(DirFree)( mmDirectory *dir MM_PARAMS );
568 #define mmDirInit(x,y,z) MM_FUNC(DirInit)(x,y,z,__FILE__,__LINE__)
569 #define mmDirSize(x,y) MM_FUNC(DirSize)(x,y,__FILE__,__LINE__)
570 #define mmDirSet(x,y,z) MM_FUNC(DirSet)(x,y,z,__FILE__,__LINE__)
571 #define mmDirGet(x,y) MM_FUNC(DirGet)(x,y,__FILE__,__LINE__)
572 #define mmDirSetFast(x,y,z) MM_FUNC(DirSetFast)(x,y,z,__FILE__,__LINE__)
573 #define mmDirGetFast(x,y) MM_FUNC(DirGetFast)(x,y,__FILE__,__LINE__)
574 #define mmDirFree(x) MM_FUNC(DirFree)(x,__FILE__,__LINE__)
584 void *MM_FUNC(AlignAlloc)( size_t bytes, intptr_t align MM_PARAMS );
585 void MM_FUNC(AlignFree)( void *v MM_PARAMS );
586 void *MM_FUNC(AlignGrow)( void *v, size_t bytes, size_t copybytes, intptr_t align MM_PARAMS );
587 void *MM_FUNC(AlignRelayAlloc)( void *(*relayalloc)( void *head, size_t bytes MM_PARAMS ), void *relayvalue, size_t bytes, intptr_t align, size_t displacement MM_PARAMS );
588 void MM_FUNC(AlignRelayFree)( void (*relayfree)( void *head, void *v, size_t bytes MM_PARAMS ), void *relayvalue, void *v, size_t bytes MM_PARAMS );
591 #define mmAlignAlloc(x,y) MM_FUNC(AlignAlloc)(x,y,__FILE__,__LINE__)
592 #define mmAlignFree(x) MM_FUNC(AlignFree)(x,__FILE__,__LINE__)
593 #define mmAlignGrow(x) MM_FUNC(AlignGrow)(x,__FILE__,__LINE__)
594 #define mmAlignRelayAlloc(v,w,x,y,z) MM_FUNC(AlignRelayAlloc)(v,w,x,y,z,__FILE__,__LINE__)
595 #define mmAlignRelayFree(w,x,y,z) MM_FUNC(AlignRelayFree)(w,x,y,z,__FILE__,__LINE__)
607 size_t volumeblocksize;
609 size_t volumechunksize;
611 size_t totalfreesize;
615 void *(*relayalloc)( void *head, size_t bytes MM_PARAMS );
616 void (*relayfree)( void *head, void *v, size_t bytes MM_PARAMS );
621 void MM_FUNC(VolumeInit)( mmVolumeHead *head, size_t volumesize, size_t minchunksize, size_t keepfreesize, size_t alignment MM_PARAMS );
622 void MM_FUNC(VolumeNodeInit)( mmVolumeHead *head, int nodeindex, size_t volumesize, size_t minchunksize, size_t keepfreesize, size_t alignment MM_PARAMS );
623 void *MM_FUNC(VolumeAlloc)( mmVolumeHead *head, size_t bytes MM_PARAMS );
624 void MM_FUNC(VolumeRelease)( mmVolumeHead *head, void *v MM_PARAMS );
625 void MM_FUNC(VolumeFree)( mmVolumeHead *head, void *v MM_PARAMS );
626 void MM_FUNC(VolumeShrink)( mmVolumeHead *head, void *v, size_t bytes MM_PARAMS );
627 size_t MM_FUNC(VolumeGetAllocSize)( mmVolumeHead *head, void *v );
628 void MM_FUNC(VolumeClean)( mmVolumeHead *head MM_PARAMS );
629 void MM_FUNC(VolumeFreeAll)( mmVolumeHead *head MM_PARAMS );
630 void *MM_FUNC(VolumeRealloc)( mmVolumeHead *head, void *v, size_t bytes MM_PARAMS );
633 #define mmVolumeInit(w,x,y,z,a) MM_FUNC(VolumeInit)(w,x,y,z,a,__FILE__,__LINE__);
634 #define mmVolumeNodeInit(v,w,x,y,z) MM_FUNC(VolumeNodeInit)(v,w,x,y,z,__FILE__,__LINE__);
635 #define mmVolumeAlloc(x,y) MM_FUNC(VolumeAlloc)(x,y,__FILE__,__LINE__);
636 #define mmVolumeRelease(x,y) MM_FUNC(VolumeRelease)(x,y,__FILE__,__LINE__);
637 #define mmVolumeFree(x,y) MM_FUNC(VolumeFree)(x,y,__FILE__,__LINE__);
638 #define mmVolumeShrink(x,y,z) MM_FUNC(VolumeShrink)(x,y,z,__FILE__,__LINE__);
639 #define mmVolumeGetAllocSize(x) MM_FUNC(VolumeGetAllocSize)(x,y,__FILE__,__LINE__);
640 #define mmVolumeClean(x) MM_FUNC(VolumeClean)(x,__FILE__,__LINE__);
641 #define mmVolumeFreeAll(x) MM_FUNC(VolumeFreeAll)(x,__FILE__,__LINE__);
642 #define mmVolumeRealloc(x) MM_FUNC(VolumeRealloc)(x,y,z,__FILE__,__LINE__);
644 #define mmVolumeAlloc MM_FUNC(VolumeAlloc)
645 #define mmVolumeRelease MM_FUNC(VolumeRelease)
646 #define mmVolumeFree MM_FUNC(VolumeFree)
651 void mmVolumeRelayByZone( mmVolumeHead *head, void *zonehead );
664 size_t pagealignment;
667 size_t chunkheadersize;
672 int MM_FUNC(ZoneInit)( mmZoneHead *head, size_t zonesize, intptr_t alignment MM_PARAMS );
673 void *MM_FUNC(ZoneAlloc)( mmZoneHead *head, size_t bytes MM_PARAMS );
674 void MM_FUNC(ZoneFree)( mmZoneHead *head, void *v MM_PARAMS );
675 void MM_FUNC(ZoneFreeAll)( mmZoneHead *head MM_PARAMS );
678 #define mmZoneInit(x,y,z) MM_FUNC(ZoneInit)(x,y,z,__FILE__,__LINE__);
679 #define mmZoneAlloc(x,y) MM_FUNC(ZoneAlloc)(x,y,__FILE__,__LINE__);
680 #define mmZoneFree(x,y) MM_FUNC(ZoneFree)(x,y,__FILE__,__LINE__);
681 #define mmZoneFreeAll(x) MM_FUNC(ZoneFreeAll)(x,__FILE__,__LINE__);
690 void *mmAlloc( void *unused, size_t bytes MM_PARAMS );
691 void *mmRealloc( void *unused, void *v, size_t bytes MM_PARAMS );
692 void mmFree( void *unused, void *v, size_t bytes MM_PARAMS );
694 void *mmDebugAlloc( size_t bytes, const char *file, int line );
695 void *mmDebugRealloc( void *v, size_t bytes, const char *file, int line );
696 void mmDebugFree( void *v, const char *file, int line );
697 #define mmDebugAlloc(x) mmDebugAlloc(x,__FILE__,__LINE__)
698 #define mmDebugFree(x) mmDebugFree(x,__FILE__,__LINE__)
699 #define mmDebugRealloc(x,y) mmDebugRealloc(x,y,__FILE__,__LINE__)
701 void mmListUses( const char *file, int line );
702 #define mmListUses() mmListUses(__FILE__,__LINE__);
709 #define malloc(x) mmAlloc(0,(x),__FILE__,__LINE__)
710 #define realloc(x,y) mmRealloc(0,(x),(y),__FILE__,__LINE__)
711 #define free(x) mmFree(0,(x),0,__FILE__,__LINE__)
712 #elif defined(MM_ALLOC_CHECK)
714 static inline void *mmAllocCheck( size_t size, const char *file, int line )
720 fprintf( stderr, "WARNING : Denied memory allocation ( %ld bytes ) at %s:%d\n", (long)size, file, line );
723 fprintf( stderr, "WARNING : Denied memory allocation ( %lld bytes ) at %s:%d\n", (long long)size, file, line );
728 static inline void *mmReallocCheck( void *p, size_t size, const char *file, int line )
730 p = realloc( p, size );
733 fprintf( stderr, "WARNING : Denied memory allocation ( %ld bytes ) at %s:%d\n", (long)size, file, line );
736 fprintf( stderr, "WARNING : Denied memory allocation ( %lld bytes ) at %s:%d\n", (long long)size, file, line );
741 #define malloc(x) mmAllocCheck((x),__FILE__,__LINE__)
742 #define realloc(x,y) mmReallocCheck((x),(y),__FILE__,__LINE__)
752 static inline uint64_t mmGetMillisecondsTime()
754 struct timeval lntime;
755 gettimeofday( &lntime, 0 );
756 return ( (uint64_t)lntime.tv_sec * 1000 ) + ( (uint64_t)lntime.tv_usec / 1000 );
760 static inline uint64_t mmGetMicrosecondsTime()
762 struct timeval lntime;
763 gettimeofday( &lntime, 0 );
764 return ( (uint64_t)lntime.tv_sec * 1000000 ) + (uint64_t)lntime.tv_usec;
768 static inline uint64_t mmGetNanosecondsTime()
770 struct timeval lntime;
771 gettimeofday( &lntime, 0 );
772 return ( (uint64_t)lntime.tv_sec * 1000000000 ) + ( (uint64_t)lntime.tv_usec * 1000 );