The Ecere SDK - ede/blob - libede/src/FileSystemCache.ec

   1 public import "ecere"
   2
   3 public import "FileSystemIterator"
   4
   5 public class FileSystemCacheIterator : InterfaceFileSystemIterator
   6 {
   7 public:
   8    FileSystemCache cache;
   9
  10    bool Iterate(char * startPath, bool followLinks)
  11    {
  12       bool result = true;
  13       char * path;
  14       FSCacheObject o;
  15       if(cache && cache.objects.count)
  16       {
  17          o = cache.FetchObject(startPath);
  18          if(o)
  19          {
  20             bool listDirEntries;
  21             FileSystemCacheIteratorStackFrame frame = null;
  22             Array<FileSystemCacheIteratorStackFrame> stack { };
  23             if(iterateStartPath)
  24             {
  25                path = o.GetPath();
  26                listDirEntries = OnObject(owner, o.name, path, o.stats, true);
  27                if(o.firstChild)
  28                {
  29                   if(listDirEntries)
  30                   {
  31                      OnEnteringDirectory(owner, path);
  32                      stack.Add((frame = { path, o.firstChild }));
  33                      path = null;
  34                   }
  35                   else
  36                      result = false;
  37                }
  38                delete path;
  39             }
  40             // else { /* TODO: frame is not initialized!! */ }
  41
  42             while(frame)
  43             {
  44                o = frame.o;
  45                if(o)
  46                {
  47                   frame.o = o.nextSibling;
  48                   path = o.GetPath();
  49                   listDirEntries = OnObject(owner, o.name, path, o.stats, !iterateStartPath && stack.count == 1);
  50                   if(o.firstChild)
  51                   {
  52                      if(listDirEntries)
  53                      {
  54                         OnEnteringDirectory(owner, path);
  55                         stack.Add((frame = { path, o.firstChild }));
  56                         path = null;
  57                      }
  58                      else
  59                         result = false;
  60                   }
  61                   delete path;
  62                }
  63                else
  64                {
  65                   if(stack.count > (iterateStartPath ? 0 : 1))
  66                      OnLeavingDirectory(owner, frame.path);
  67                   delete frame;
  68                   stack.lastIterator.Remove();
  69                   frame = stack.count == 0 ? null : stack.lastIterator.data;
  70                }
  71             }
  72             if(stack.count != 0)
  73                PrintLn("dddddddd");
  74             delete stack;
  75          }
  76       }
  77       return result;
  78    }
  79 }
  80
  81 class FileSystemCacheIteratorStackFrame : struct
  82 {
  83 public:
  84    String path;
  85    FSCacheObject o;
  86
  87 private:
  88    ~FileSystemCacheIteratorStackFrame()
  89    {
  90       delete path;
  91    }
  92 };
  93
  94 class FileSystemCacheBits
  95 {
  96    bool indexInodes:1;
  97    bool indexByName:1;
  98    bool indexBySize:1;
  99 }
 100
 101 public class DummyFileSystemCacheWindow : Window
 102 {
 103 public:
 104    FileSystemCache cache { };
 105 }
 106
 107 public class FileSystemCache
 108 {
 109 public:
 110    Map<FSCacheObject, FSCacheRoot> roots { };
 111    Array<FSCacheObject> objects { };
 112    Array<FSCacheObject> normalParentStack;
 113    Map<uint, FileSystemDeviceCache> deviceCaches { };
 114
 115    Map<String, Array<FSCacheObject>> nameIndex { };
 116    Map<String, bool> nonSingleNames { };
 117    Map<FileSize, Array<FSCacheObject>> sizeIndex { };
 118    Map<FileSize, bool> nonSingleSizes { };
 119
 120    property bool indexInodes { set { bits.indexInodes = value; } get { return bits.indexInodes; } };
 121    property bool indexByName { set { bits.indexByName = value; } get { return bits.indexByName; } };
 122    property bool indexBySize { set { bits.indexBySize = value; } get { return bits.indexBySize; } };
 123
 124    void/*FileSystemCache*/ ::Cache(char * path, bool indexInodes, bool indexByName, bool indexBySize, DummyFileSystemCacheWindow dummyWindow)
 125    {
 126       FileSystemCache cache = dummyWindow.cache;
 127       // TOIMP: for now we will assume the cache includes files from a single domain (partition/share)
 128       // TOIMP: would like to use full blown file iterator driver system to support other types of file system, archives, etc
 129       FileAttribs pathAttribs = FileExists(path);
 130       if(pathAttribs)
 131       {
 132          cache = dummyWindow.cache;// = { };
 133          cache.indexInodes = indexInodes;
 134          cache.indexByName = indexByName;
 135          cache.indexBySize = indexBySize;
 136          cache.normalParentStack = { };
 137          //cache.domainParentStack = { };
 138          if(pathAttribs.isDirectory)
 139          {
 140             FileSystemIterator fsi
 141             {
 142                owner = dummyWindow; //owner = (void *)this;
 143                iterateStartPath = true;
 144
 145                // COMPILER TOFIX: this will not compile (error: method class must be derived from class) -- bool Whatever/*FileSystemCache*/::OnFolder(...)
 146                bool /*FileSystemCache*/DummyFileSystemCacheWindow::OnObject(char * name, char * path, FileStats stats, bool isRootObject)
 147                {
 148                   bool result = true;
 149                   FileSystemCache cache = this.cache;
 150                   uint dev = 0;//stats.dev;
 151                   uint inode = 0;//stats.inode;
 152                   char * p;
 153                   FSCacheObject parent = isRootObject ? null : cache.normalParentStack.lastIterator.data;
 154                   FSCacheObject o { parent, name = CopyString(isRootObject ? path : name), stats = stats };
 155                   FileStats s;
 156                   if(isRootObject)
 157                   {
 158                      // TODO see if we can find a parent for it
 159                      // see if we already have a FSCacheObject for it, if so, update
 160                      // distant parents -- when there are gaps.. maybe the parent's parent is present but not the parent
 161                      // keep a list of those and see if they can be pluged in after a Cache run
 162                      cache.roots[o] = FSCacheRoot { cache.objects.count, GetPathDirNamesArray(path) };
 163                   }
 164                   else
 165                   {
 166                      if(!parent.firstChild)
 167                         parent.firstChild = parent.lastChild = o;
 168                      else
 169                      {
 170                         parent.lastChild.nextSibling = o;
 171                         o.prevSibling = parent.lastChild;
 172                         parent.lastChild = o;
 173                      }
 174                   }
 175                   cache.objects.Add(o);
 176 #if _DEBUG
 177                   if(cache.objects.count % 1000 == 0)
 178                      PrintLn(path, " ----- ", cache.objects.count / 1000, " --------- ", dev);
 179 #if 0
 180                   FileGetStatsLink(path, s);
 181                   if(s.inode != inode)
 182                   {
 183                      PrintLn(s.inode);
 184                      PrintLn(inode);
 185                      PrintLn("crap");
 186                   }
 187                   p = o.GetPath();
 188                   if(strcmp(p, path))
 189                   {
 190                      int c;
 191                      PrintLn("damn");
 192                      PrintLn(p);
 193                      for(c = 0; c < cache.normalParentStack.count; c++)
 194                      {
 195                         delete p;
 196                         p = cache.normalParentStack[c].GetPath();
 197                         PrintLn(p);
 198                      }
 199                      PrintLn(path);
 200                      PrintLn("bad");
 201                   }
 202                   delete p;
 203 #endif
 204 #endif
 205                   if(!stats.attribs.isDirectory)
 206                   {
 207                      if(dev && inode && cache.bits.indexInodes)
 208                      {
 209                         FileSystemDeviceCache devCache = cache.deviceCaches[dev];
 210                         if(!devCache)
 211                            cache.deviceCaches[dev] = devCache = { };
 212                         {
 213                            Array<FSCacheObject> files = devCache.inodes[inode];
 214                            if(!files)
 215                            {
 216                               devCache.inodes[inode] = files = { };
 217                               if(cache.bits.indexBySize)
 218                                  CacheAddFileToIndexBySize(cache, o, stats.size);
 219                            }
 220                            else if(files.count == 1)
 221                               devCache.nonSingleLinks[inode] = true;
 222                            files.Add(o);
 223                         }
 224                      }
 225                      else if(cache.bits.indexBySize)
 226                         CacheAddFileToIndexBySize(cache, o, stats.size);
 227                      if(cache.bits.indexByName)
 228                         CacheAddFileToIndexByName(cache, o, name);
 229                   }
 230                   return result;
 231                }
 232
 233                void /*FileSystemCache*/DummyFileSystemCacheWindow::OnEnteringDirectory(char * path)
 234                {
 235                   FileSystemCache cache = this.cache;
 236                   FSCacheObject o = cache.objects.lastIterator.data;
 237                   cache.normalParentStack.Add(o);
 238                }
 239
 240                void /*FileSystemCache*/DummyFileSystemCacheWindow::OnLeavingDirectory(char * path)
 241                {
 242                   FileSystemCache cache = this.cache;
 243                   cache.normalParentStack.lastIterator.Remove();
 244                }
 245             };
 246             fsi.Iterate(path, false);
 247             {
 248                if(cache.objects.count)
 249                {
 250                   for(dev : cache.deviceCaches)
 251                   {
 252                      PrintLn("# of inodes (dev", &dev, "): ", dev.inodes.count);
 253
 254                      if(dev.nonSingleLinks.count)
 255                      {
 256                         for(inode : dev.nonSingleLinks)
 257                         {
 258                            PrintLn("   ", &inode);
 259                            for(o : dev.inodes[&inode])
 260                            {
 261                               char * path = o.GetPath();
 262                               PrintLn(path);
 263                               delete path;
 264                            }
 265                         }
 266                      }
 267                   }
 268                }
 269             }
 270             delete cache.normalParentStack;
 271             delete fsi;
 272          }
 273       }
 274       //return cache;
 275    }
 276
 277    FSCacheObject FetchObject(char * path)
 278    {
 279       FSCacheObject result = null;
 280       for(root : roots)
 281       {
 282          FSCacheObject o = &root;
 283          Array<String> names = GetPathDirNamesArray(path);
 284          int c, max = Min(names.count, root.names.count);
 285          for(c = 0; c < max; c++)
 286          {
 287             if(strcmp(names[c], root.names[c]))
 288                break;
 289          }
 290          if(c == max)
 291          {
 292             if(c == names.count)
 293                result = o;
 294             else
 295             {
 296                for(; c < names.count; c++)
 297                {
 298                   FSCacheObject child = o.firstChild;
 299                   o = null;
 300                   while(child)
 301                   {
 302                      if(!strcmp(child.name, names[c]))
 303                      {
 304                         o = child;
 305                         break;
 306                      }
 307                      child = child.nextSibling;
 308                   }
 309                   if(!o)
 310                      break;
 311                }
 312                if(o)
 313                   result = o;
 314             }
 315             break;
 316          }
 317          delete names;
 318       }
 319       return result;
 320    }
 321
 322    bool FileGetStats(char * path, FileStats stats)
 323    {
 324       FSCacheObject o = FetchObject(path);
 325       if(o)
 326          stats = o.stats;
 327       else
 328          stats = { };
 329       return o != null;
 330    }
 331
 332    Map<uint, Map<uint, bool>> devsInodesDone { };
 333    Map<String, bool> linksPaths { };
 334 #if 0
 335    void Special(char * path/*, Map<String, bool> linksPaths*//*Map<uint, Map<uint, bool>> devsInodesDone*/, DummyFileSystemCacheWindow dummyWindow)
 336    {
 337       FileSystemCacheIterator fsci { owner = dummyWindow, cache = this, iterateStartPath = true;
 338          bool /*FileSystemCache*/DummyFileSystemCacheWindow::OnObject(char * name, char * path, FileStats stats, bool isRootObject)
 339          {
 340             uint dev = stats.dev;
 341             uint inode = stats.inode;
 342             FileSystemCache cache = this.cache;
 343             if(dev && inode)
 344             {
 345                FileSystemDeviceCache devCache = cache.deviceCaches[dev];
 346                if(devCache)
 347                {
 348                   Array<FSCacheObject> links = devCache.inodes[inode];
 349                   if(links && links.count)
 350                   {
 351                      if(links.count > 1)
 352                      {
 353                         Map<uint, bool> inodesDone = cache.devsInodesDone[dev];
 354                         Map<String, bool> linksPaths = cache.linksPaths;
 355                         bool done;
 356                         if(!inodesDone)
 357                            cache.devsInodesDone[dev] = inodesDone = { };
 358                         done = inodesDone[inode];
 359                         if(!done)
 360                         {
 361                            inodesDone[inode] = true;
 362                            //PrintLn("   ", &inode);
 363                            for(o : links)
 364                            {
 365                               char * path = o.GetPath();
 366                               //PrintLn(path);
 367                               //delete path;
 368                               linksPaths[path] = true;
 369                            }
 370                         }
 371                      }
 372                   }
 373                   else
 374                      PrintLn("no links/count!!");
 375                }
 376                else
 377                   PrintLn("no devCache!!");
 378             }
 379             else
 380                PrintLn("no dev/inode");
 381             return true;
 382          }
 383       };
 384       fsci.Iterate(path, false);
 385       delete fsci;
 386    }
 387    void SpecialPrint()
 388    {
 389       MapNode<String, bool> mn;
 390       PrintLn(" -------------------------------------------- Special -------------------------------------------- ");
 391       PrintLn(" ------------------------------------------------------------------------------------------------- ");
 392       if(linksPaths.count)
 393       {
 394          for(mn = linksPaths.root.minimum; mn; mn = mn.next)
 395          {
 396             PrintLn(mn.key);
 397          }
 398       }
 399       else
 400          PrintLn("   no hard linked files");
 401       PrintLn(" ------------------------------------------------------------------------------------------------- ");
 402    }
 403 #endif
 404    void Special(char * path/*, Map<String, bool> linksPaths*//*Map<uint, Map<uint, bool>> devsInodesDone*/, DummyFileSystemCacheWindow dummyWindow)
 405    {
 406    }
 407    void SpecialPrint()
 408    {
 409       MapNode<FileSize, bool> mn;
 410       PrintLn(" -------------------------------------------- Special -------------------------------------------- ");
 411       PrintLn(" ------------------------------------------------------------------------------------------------- ");
 412       if(nonSingleSizes.count)
 413       {
 414          for(mn = nonSingleSizes.root.minimum; mn; mn = mn.next)
 415          {
 416             Array<FSCacheObject> files = sizeIndex[mn.key];
 417             PrintLn("     size: ", mn.value, "# of files: ", files.count);
 418             for(o : files)
 419             {
 420                char * p = o.GetPath();
 421                PrintLn(p);
 422                delete p;
 423             }
 424          }
 425       }
 426       PrintLn(" ------------------------------------------------------------------------------------------------- ");
 427    }
 428
 429 private:
 430    FileSystemCacheBits bits;
 431
 432    ~FileSystemCache()
 433    {
 434       objects.Free();
 435       delete objects;
 436    }
 437
 438 }
 439
 440 static inline void CacheAddFileToIndexBySize(FileSystemCache cache, FSCacheObject o, FileSize size)
 441 {
 442    Array<FSCacheObject> files = cache.sizeIndex[size];
 443    if(!files)
 444       cache.sizeIndex[size] = files = { };
 445    else if(files.count == 1)
 446       cache.nonSingleSizes[size] = true;
 447    files.Add(o);
 448 }
 449
 450 static inline void CacheAddFileToIndexByName(FileSystemCache cache, FSCacheObject o, char * name)
 451 {
 452    Array<FSCacheObject> files = cache.nameIndex[name];
 453    if(!files)
 454       cache.nameIndex[name] = files = { };
 455    else if(files.count == 1)
 456       cache.nonSingleNames[name] = true;
 457    files.Add(o);
 458 }
 459
 460
 461 public class FileSystemDeviceCache : struct
 462 {
 463 public:
 464    Map<uint, Array<FSCacheObject>> inodes { };
 465    Map<uint, bool> nonSingleLinks { }; // using a map presuming it will be faster than Array::Find() //Array<uint> nonSingleLinks { };
 466
 467 private:
 468 }
 469
 470 class FSCacheRoot : struct
 471 {
 472 public:
 473    uint position;
 474    Array<String> names;
 475
 476 private:
 477    ~FSCacheRoot()
 478    {
 479       if(names)
 480       {
 481          names.Free();
 482          delete names;
 483       }
 484    }
 485 }
 486
 487 /*
 488 enum FSCacheObjectType { normal };
 489
 490 class FSCacheObjectBits
 491 {
 492    FSCacheObjectType type:4;
 493    bool isRoot:1;
 494 }
 495 */
 496
 497 public class FSCacheObject : struct
 498 {
 499 public:
 500    FSCacheObject parent;
 501    char * name;
 502    FileStats stats;
 503    FSCacheObject firstChild;
 504    FSCacheObject lastChild;
 505    FSCacheObject prevSibling;
 506    FSCacheObject nextSibling;
 507
 508    char * GetPath()
 509    {
 510       int c;
 511       char path[MAX_LOCATION] = "";
 512       FSCacheObject o;
 513       Array<FSCacheObject> stack { };
 514       for(o = this; o; o = o.parent)
 515          stack.Add(o);
 516       for(c = stack.count - 1; c >= 0; c--)
 517       {
 518          o = stack[c];
 519          if(c < stack.count - 1)
 520             strcat(path, DIR_SEPS);
 521          strcat(path, o.name);
 522       }
 523       delete stack;
 524       return CopyString(path);
 525    }
 526
 527 private:
 528 //   FSCacheObjectBits bits;
 529
 530    ~FSCacheObject()
 531    {
 532       delete name;
 533    }
 534 }
 535
 536 Array<String> GetPathDirNamesArray(char * path)
 537 {
 538    Array<String> names;
 539    if(path && path[0])
 540    {
 541       char * p = path;
 542       char rest[MAX_LOCATION];
 543       char name[MAX_FILENAME];
 544       names = { };
 545       while(strlen(p))
 546       {
 547          SplitDirectory(p, name, rest);
 548          if(p == path) p = rest;
 549          names.Add(CopyString(name));
 550       }
 551    }
 552    return names;
 553 }