The Ecere SDK - sdk/blob - ecere/src/net/HTTPFile.ec

   1 #ifndef ECERE_NONET
   2
   3 #include <stdio.h>
   4
   5 import "List"
   6 import "network"
   7
   8 class ConnectionsHolder
   9 {
  10    List<HTTPConnection> connections { };
  11
  12    ~ConnectionsHolder()
  13    {
  14       HTTPConnection c;
  15       while((c = connections[0]))
  16          delete c;   // The HTTPConnection destructor will take out from the list
  17    }
  18 }
  19
  20 static ConnectionsHolder holder { };
  21
  22 namespace net;
  23
  24 /*static */define HTTPFILE_BUFFERSIZE = 65536;
  25
  26 static Mutex connectionsMutex { };
  27
  28 static class ServerNode : BTNode
  29 {
  30    char address[24];
  31    bool resolved;
  32    ~ServerNode()
  33    {
  34       char * name = (char *)key;
  35       delete name;
  36    }
  37 }
  38
  39 static class ServerNameCache
  40 {
  41    BinaryTree servers
  42    {
  43       CompareKey = (void *)BinaryTree::CompareString;
  44    };
  45    Mutex mutex { };
  46
  47    ~ServerNameCache()
  48    {
  49       ServerNode server;
  50       while((server = (ServerNode)servers.root))
  51       {
  52          servers.Remove(server);
  53          delete server;
  54       }
  55    }
  56    bool Resolve(char * host, char * address)
  57    {
  58       ServerNode server;
  59       mutex.Wait();
  60       server = (ServerNode)servers.FindString(host);
  61       if(!server)
  62       {
  63          server = ServerNode { key = (uintptr)CopyString(host) };
  64          servers.Add(server);
  65          server.resolved = GetAddressFromName(host, server.address);
  66       }
  67       mutex.Release();
  68       if(server.resolved)
  69          strcpy(address, server.address);
  70       return server.resolved;
  71    }
  72 };
  73
  74 static ServerNameCache serverNameCache { };
  75
  76 static char * GetString(char * string, char * what, int count)
  77 {
  78    int c;
  79    for(c = 0; what[c]; c++)
  80    {
  81       if((count && c >= count) || (string[c] != what[c] && tolower(string[c]) != tolower(what[c])))
  82          return null;
  83    }
  84    return string + c;
  85 }
  86
  87 private class HTTPConnection : Socket
  88 {
  89    class_no_expansion;
  90    char * server;
  91    int port;
  92    HTTPFile file;
  93
  94    processAlone = true;
  95
  96    ~HTTPConnection()
  97    {
  98       // printf("Before TakeOut we have %d connections:\n", holder.connections.count); for(c : holder.connections) ::PrintLn(c.server); ::PrintLn("");
  99       holder.connections.TakeOut(this);
 100       /*
 101       PrintLn(server, " Connection Closed (", holder.connections.count, ") opened");
 102       printf("Now we have %d connections:\n", holder.connections.count);
 103       for(c : holder.connections)
 104       {
 105          ::PrintLn(c.server);
 106       }
 107       ::PrintLn("");
 108       */
 109       delete server;
 110    }
 111
 112    void OnDisconnect(int code)
 113    {
 114       connectionsMutex.Wait();
 115       if(file)
 116          delete file.connection; // This decrements the file's reference
 117
 118       connectionsMutex.Release();
 119
 120       // PrintLn(server, " Disconnected Us");
 121
 122       delete this;         // The 'connections' reference
 123    }
 124
 125    uint Open_OnReceive(const byte * buffer, uint count)
 126    {
 127       HTTPFile file = this.file;
 128       int pos = 0;
 129       int c;
 130       while(!file.done)
 131       {
 132          bool gotEndLine = false;
 133          for(c = 0; c<(int)count-1; c++)
 134          {
 135             if(buffer[c] == '\r' && buffer[c+1] == '\n')
 136             {
 137                gotEndLine = true;
 138                break;
 139             }
 140          }
 141          if(!gotEndLine)
 142             // Incomplete packet
 143             return pos;
 144          if(c<count)
 145          {
 146             char * string = (char *)buffer;
 147
 148 #ifdef _DEBUG
 149             fwrite(buffer, 1, c, stdout);
 150             puts("");
 151 #endif
 152
 153             if(!c)
 154             {
 155                //if(file.openStarted)
 156                   file.done = true;
 157             }
 158             else
 159             {
 160                //file.openStarted = true;
 161                if((string = GetString((char *)buffer, "HTTP/1.1 ", count)) ||
 162                   (string = GetString((char *)buffer, "HTTP/1.0 ", count)))
 163                {
 164                   file.status = atoi(string);
 165                }
 166                else if(string = GetString((char *)buffer, "Transfer-Encoding: ", count))
 167                {
 168                   if(!strnicmp(string, "chunked", strlen("chunked")))
 169                   {
 170                      file.chunked = true;
 171                   }
 172                }
 173                else if(string = GetString((char *)buffer, "Content-Length: ", count))
 174                {
 175                   file.totalSize = atoi(string);
 176                   file.totalSizeSet = true;
 177                }
 178                else if(string = GetString((char *)buffer, "Content-Type: ", count))
 179                {
 180                   char * cr = strstr(string, "\r");
 181                   char * lf = strstr(string, "\n");
 182                   int len;
 183                   if(cr)
 184                      len = cr - string;
 185                   else if(lf)
 186                      len = lf - string;
 187                   else
 188                      len = strlen(string);
 189
 190                   file.contentType = new char[len+1];
 191                   memcpy(file.contentType, string, len);
 192                   file.contentType[len] = 0;
 193                }
 194                else if(string = GetString((char *)buffer, "Content-disposition: ", count))
 195                {
 196                   char * cr = strstr(string, "\r");
 197                   char * lf = strstr(string, "\n");
 198                   int len;
 199                   if(cr)
 200                      len = cr - string;
 201                   else if(lf)
 202                      len = lf - string;
 203                   else
 204                      len = strlen(string);
 205
 206                   file.contentDisposition = new char[len+1];
 207                   memcpy(file.contentDisposition, string, len);
 208                   file.contentDisposition[len] = 0;
 209                }
 210                else if(string = GetString((char *)buffer, "Connection: ", count))
 211                {
 212                   if(!strnicmp(string, "close", strlen("close")))
 213                   {
 214                      file.close = true;
 215                   }
 216                }
 217                else if(string = GetString((char *)buffer, "Location: ", count))
 218                {
 219                   if(file.relocation)
 220                   {
 221                      strncpy(file.relocation, buffer + 10, c - 10);
 222                      file.relocation[c - 10] = '\0';
 223                   }
 224                }
 225             }
 226             // return c+2;
 227             if(buffer[c] == '\r') c++;
 228             if(buffer[c] == '\n') c++;
 229             pos += c;
 230             count -= c;
 231             buffer += c;
 232
 233          }
 234          else
 235             break;
 236       }
 237
 238       return pos;
 239    }
 240
 241    uint Read_OnReceive(const byte * buffer, uint count)
 242    {
 243       HTTPFile file = this.file;
 244       if(file)
 245       {
 246          int read;
 247          if(file.chunked && !file.chunkSize)
 248          {
 249             int pos = 0, c = 0;
 250             char * string = null;
 251             bool ready = false;
 252             while(pos + c < (int)count-3)
 253             {
 254                if(buffer[pos+c] == '\r' && buffer[pos+c+1] == '\n')
 255                {
 256                   if(string)
 257                   {
 258                      ready = true;
 259                      pos += c + 2;
 260                      if(buffer[pos] == '\r' && buffer[pos+1] =='\n') pos+= 2;
 261                      break;
 262                   }
 263                   else
 264                   {
 265                      pos += 2;
 266                      c = 0;
 267                   }
 268                }
 269                else
 270                {
 271                   if(!string)
 272                      string = buffer + pos;
 273                   c++;
 274                }
 275             }
 276             if(ready)
 277             {
 278                file.chunkSize = strtol(string, null, 16);
 279                if(!file.chunkSize)
 280                {
 281                   file.connection.file = null;
 282                   delete file.connection; // This decrements the file's reference
 283                }
 284             }
 285             return pos;
 286          }
 287
 288          read = Min(count, HTTPFILE_BUFFERSIZE - file.bufferCount);
 289          if(file.chunked)
 290          {
 291             read = Min(read, file.chunkSize);
 292             file.chunkSize -= read;
 293          }
 294          if(read)
 295          {
 296             memcpy(file.buffer + file.bufferCount, buffer, read);
 297             file.bufferCount += read;
 298          }
 299          return read;
 300       }
 301       else
 302          return count;
 303    }
 304 }
 305
 306 public class HTTPFile : File
 307 {
 308    bool reuseConnection;
 309    reuseConnection = true;
 310 public:
 311    property bool reuseConnection
 312    {
 313       set { reuseConnection = value; }
 314       get { return reuseConnection; }
 315    }
 316    property String contentType
 317    {
 318       get { return contentType; }
 319    }
 320    property String contentDisposition
 321    {
 322       get { return contentDisposition; }
 323    }
 324
 325    bool OpenURL(char * name, char * referer, char * relocation)
 326    {
 327       return RetrieveHead(name, referer, relocation, false);
 328    }
 329
 330 private:
 331
 332    bool RetrieveHead(char * name, char * referer, char * relocation, bool askBody)
 333    {
 334       bool result = false;
 335       String http;
 336       if(!this || !name) return false;
 337       http = strstr(name, "http://");
 338       if(http != name) http = null;
 339
 340       askedBody = askBody;
 341
 342       done = false;
 343       delete contentType;
 344       delete contentDisposition;
 345       // ::PrintLn("Opening ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID(), "\n");
 346       if(this && http)
 347       {
 348          char server[1024];
 349          char msg[1024];
 350          int len;
 351          char * serverStart = ecere::net::GetString(name, "http://", 0);
 352          char * fileName = strstr(serverStart, "/");
 353          int port = 80;
 354          char * colon;
 355          bool reuse = false;
 356          HTTPConnection connection = null;
 357          close = false;
 358
 359          if(fileName)
 360          {
 361             fileName++;
 362             memcpy(server, serverStart, fileName - serverStart - 1);
 363             server[fileName - serverStart - 1] = '\0';
 364          }
 365          else
 366             strcpy(server, serverStart);
 367
 368          if(relocation && !fileName && name[strlen(name)-1] != '/')
 369          {
 370             strcpy(relocation, "http://");
 371             strcat(relocation, server);
 372             strcat(relocation, "/");
 373          }
 374
 375          colon = strchr(server, ':');
 376          if(colon)
 377          {
 378             port = atoi(colon+1);
 379             *colon = '\0';
 380          }
 381
 382          connectionsMutex.Wait();
 383          if(this.connection)
 384          {
 385             this.connection.file = null;
 386             if(close)
 387                this.connection.Disconnect(0);
 388             delete this.connection;
 389          }
 390
 391          if(chunked)
 392          {
 393             while(this.connection && this.connection.connected && this.connection.file)
 394             {
 395                connectionsMutex.Release();
 396                this.connection.Process();
 397                connectionsMutex.Wait();
 398             }
 399          }
 400
 401          if(reuseConnection)
 402          {
 403             bool retry = true;
 404             while(retry)
 405             {
 406                retry = false;
 407                connection = null;
 408                for(c : holder.connections)
 409                {
 410                   if(!strcmpi(c.server, server) && c.port == port)
 411                   {
 412                      if(!c.file && c.connected)
 413                      {
 414                         connection = c;      // TOFIX: 'incref c' doesn't work
 415                         incref connection;      // HTTPFile reference if we keep it
 416                         connectionsMutex.Release();
 417                         connection.ProcessTimeOut(0.000001);
 418                         connectionsMutex.Wait();
 419                         if(!connection.connected || connection.file)
 420                         {
 421                            // We're disconnected or reused already...
 422                            retry = true;
 423                            delete connection;
 424                         }
 425                         break;
 426                      }
 427                   }
 428                }
 429             }
 430             if(connection)
 431             {
 432                // ::PrintLn("Reusing Connection ", (uint64)connection, " for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID(), "\n");
 433                reuse = true;
 434                connection.file = this;
 435             }
 436          }
 437
 438       tryagain:
 439          if(!connection)
 440          {
 441             char ipAddress[1024];
 442             connection = HTTPConnection { };
 443             incref connection;      // HTTPFile reference on success
 444
 445             connection.file = this;
 446
 447             connectionsMutex.Release();
 448
 449             if(serverNameCache.Resolve(server, ipAddress))
 450             {
 451                // ::PrintLn("No Connection - Connecting ", (uint64)connection, " for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID());
 452                if(connection.Connect(ipAddress /*server*/, port))
 453                {
 454                   //::PrintLn("Successfully Connected for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID());
 455                   //::PrintLn("Waiting on connectionsMutex for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID());
 456                   connectionsMutex.Wait();
 457
 458                   connection.server = CopyString(server);
 459                   connection.port = port;
 460
 461                   //::PrintLn("Got connectionsMutex for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID());
 462                   holder.connections.Add(connection);
 463                   /*
 464                   printf("Now we have %d connections:\n", holder.connections.count);
 465                   for(c : holder.connections)
 466                   {
 467                      String s = c.server;
 468                      ::Print("Server: ");
 469                      ::PrintLn(c.server);
 470                   }
 471                   ::PrintLn("");
 472                   */
 473                   incref connection;   // Global List Reference
 474                }
 475                else
 476                {
 477                   // ::PrintLn("Connection Failed for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID());
 478                   connectionsMutex.Wait();
 479                   delete connection;
 480                }
 481             }
 482             else
 483             {
 484                connectionsMutex.Wait();
 485                delete connection;
 486             }
 487          }
 488
 489          if(connection)
 490          {
 491             incref connection;      // local reference
 492
 493             connection.OnReceive = HTTPConnection::Open_OnReceive;
 494             connection.file = this;
 495             this.connection = connection;
 496             this.relocation = relocation;
 497             //openStarted = false;
 498
 499             totalSizeSet = false;   // HEAD will sometimes give you 0!
 500             strcpy(msg, askBody ? "GET /" : "HEAD /");
 501
 502             if(fileName)
 503             {
 504                byte ch;
 505                int c;
 506                len = strlen(msg);
 507                for(c = 0; (ch = fileName[c]); c++)
 508                {
 509                   if(ch <= 32 || ch > 128)
 510                   {
 511                      byte nibble;
 512                      msg[len++] = '%';
 513                      nibble = (ch & 0xF0) >> 4;
 514                      msg[len++] = (nibble > 9) ? (nibble - 10 + 'a') : (nibble + '0');
 515                      nibble = ch & 0x0F;
 516                      msg[len++] = (nibble > 9) ? (nibble - 10 + 'a') : (nibble + '0');
 517                   }
 518                   else
 519                      msg[len++] = ch;
 520                }
 521                msg[len] = '\0';
 522             }
 523
 524             strcat(msg, " HTTP/1.1\r\nHost: ");
 525             //strcat(msg, " HTTP/1.0\r\nHost: ");
 526             strcat(msg, server);
 527             strcat(msg, "\r\n");
 528             strcat(msg, "Accept-Charset: UTF-8\r\n");
 529             //strcat(msg, "Accept-Charset: ISO-8859-1\r\n");
 530             strcat(msg, "Connection: Keep-Alive\r\n");
 531             if(referer)
 532             {
 533                strcat(msg, "Referer: ");
 534                strcat(msg, referer);
 535                strcat(msg, "\r\n");
 536             }
 537             strcat(msg, "\r\n");
 538             len = strlen(msg);
 539
 540             //::PrintLn("Releasing connectionsMutex before GET for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID());
 541             connectionsMutex.Release();
 542
 543             // ::PrintLn("Sending GET for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID(), "\n");
 544             connection.Send(msg, len);
 545
 546             while(this.connection && this.connection.connected && !done)
 547             {
 548                this.connection.Process();
 549             }
 550             //::PrintLn("Got DONE for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID(), "\n");
 551
 552             if(this.connection)
 553             {
 554                if(name != location)
 555                {
 556                   delete location;
 557                   location = CopyString(name);
 558                }
 559                if(status == 200 || (!status && totalSizeSet))
 560                {
 561                   if(askBody)
 562                      this.connection.OnReceive = HTTPConnection::Read_OnReceive;
 563
 564                   result = true;
 565                   connectionsMutex.Wait();
 566                }
 567                else
 568                {
 569                   if(askBody)
 570                   {
 571                      if(chunked)
 572                      {
 573                         bool wait = false;
 574                         this.connection.OnReceive = HTTPConnection::Read_OnReceive;
 575                         while(!eof)
 576                         {
 577                            if(!this.connection)
 578                               eof = true;
 579                            else
 580                            {
 581                               // First time check if we already have bytes, second time wait for an event
 582                               this.connection.Process();
 583                               wait = true;
 584                            }
 585                         }
 586                      }
 587                      else if(totalSizeSet)
 588                      {
 589                         done = false;
 590                         this.connection.OnReceive = HTTPConnection::Read_OnReceive;
 591                         while(this.connection && this.connection.connected && position < totalSize)
 592                         {
 593                            connection.Process();
 594                            position += bufferCount;
 595                            bufferCount = 0;
 596                         }
 597                      }
 598                   }
 599
 600                   connectionsMutex.Wait();
 601                   if(this.connection)
 602                   {
 603                      this.connection.OnReceive = null;
 604                      this.connection.file = null;
 605
 606                      if(close)
 607                      {
 608                         this.connection.Disconnect(0);
 609                         connection = null;
 610                      }
 611                   }
 612
 613                   status = 0;
 614                   delete this.connection; // This decrements the file's reference
 615                   this.relocation = null;
 616                   totalSize = 0;
 617                   totalSizeSet = false;
 618                   done = false;
 619                   eof = false;
 620                   position = 0;
 621                   bufferPos = 0;
 622                   bufferCount = 0;
 623                   chunked = false;
 624                }
 625             }
 626             else
 627             {
 628                connectionsMutex.Wait();
 629             }
 630             if(reuse && !status && connection && !connection.connected)
 631             {
 632                delete connection;
 633                reuse = false;
 634                goto tryagain;
 635             }
 636             else
 637                delete connection;
 638          }
 639          connectionsMutex.Release();
 640       }
 641       return result;
 642    }
 643
 644    ~HTTPFile()
 645    {
 646       delete location;
 647       delete contentType;
 648       delete contentDisposition;
 649       {
 650          connectionsMutex.Wait();
 651          if(connection)
 652          {
 653             if(totalSizeSet && askedBody)
 654             {
 655                done = false;
 656                this.connection.OnReceive = HTTPConnection::Read_OnReceive;
 657                while(this.connection && this.connection.connected && position + (bufferCount - bufferPos) < totalSize)
 658                {
 659                   connectionsMutex.Release();
 660                   connection.Process();
 661                   connectionsMutex.Wait();
 662                   position += bufferCount;
 663                   bufferCount = 0;
 664                }
 665                position = 0;
 666             }
 667
 668             if(connection)
 669             {
 670                connection.file = null;
 671                if(close)
 672                   connection.Disconnect(0);
 673                delete connection;
 674             }
 675          }
 676          connectionsMutex.Release();
 677
 678          if(chunked)
 679          {
 680             while(connection && connection.connected && connection.file)
 681             {
 682                connectionsMutex.Release();
 683                connection.Process();
 684                connectionsMutex.Wait();
 685             }
 686          }
 687          //::PrintLn("Done with ", (uint64)this);
 688       }
 689    }
 690
 691    int Read(byte * buffer, uint size, uint count)
 692    {
 693       uint readSize = size * count;
 694       uint read = 0;
 695       bool wait = false;
 696       Time lastTime = GetTime();
 697
 698       if(!askedBody)
 699       {
 700          askedBody = true;
 701          if(!RetrieveHead(this.location, null, null, true))
 702             return 0;
 703       }
 704
 705       if(totalSizeSet && position >= totalSize)
 706          eof = true;
 707       while(!eof && read < readSize && !aborted)
 708       {
 709          uint numbytes = bufferCount - bufferPos;
 710          numbytes = Min(numbytes, readSize - read);
 711          if(totalSizeSet)
 712             numbytes = Min(numbytes, totalSize - position);
 713
 714          if(numbytes)
 715          {
 716             lastTime = GetTime();
 717             memcpy(buffer + read, this.buffer + bufferPos, numbytes);
 718             bufferPos += numbytes;
 719             position += numbytes;
 720             read += numbytes;
 721
 722             lastTime = GetTime();
 723
 724             if(bufferPos > HTTPFILE_BUFFERSIZE / 2)
 725             {
 726                // Shift bytes back to beginning of buffer
 727                uint shift = bufferCount - bufferPos;
 728                if(shift)
 729                   memcpy(this.buffer, this.buffer + bufferPos, shift);
 730                bufferCount -= bufferPos;
 731                bufferPos = 0;
 732             }
 733          }
 734          else
 735          {
 736             if(!connection || !connection.connected)
 737                eof = true;
 738             else
 739             {
 740                // First time check if we already have bytes, second time wait for an event
 741                connection.Process();
 742                if(wait && bufferCount - bufferPos == 0 && GetTime() - lastTime > 5)
 743                   eof = true;
 744                wait = true;
 745             }
 746          }
 747          if(totalSizeSet && position >= totalSize)
 748             eof = true;
 749       }
 750       return read / size;
 751    }
 752
 753    int Write(byte * buffer, uint size, uint count)
 754    {
 755       return 0;
 756    }
 757
 758    bool Getc(char * ch)
 759    {
 760       int read = Read(ch, 1, 1);
 761       return !eof && read != 0;
 762    }
 763
 764    bool Putc(char ch)
 765    {
 766       return false;
 767    }
 768
 769    bool Puts(char * string)
 770    {
 771       return false;
 772    }
 773
 774    bool Seek(int pos, FileSeekMode mode)
 775    {
 776       if(mode == start && bufferPos == 0 && pos <= bufferCount & pos >= 0)
 777       {
 778          bufferPos = pos;
 779          return true;
 780       }
 781       else if(mode == current && bufferPos == 0 && (position + pos) <= bufferCount & (position + pos) >= 0)
 782       {
 783          bufferPos = position + pos;
 784          return true;
 785       }
 786       else if(mode == end && totalSizeSet && bufferPos == 0 && bufferCount == totalSize && (totalSize - pos) <= bufferCount & (totalSize - pos) >= 0)
 787       {
 788          bufferPos = totalSize - pos;
 789          return true;
 790       }
 791       return false;
 792    }
 793
 794    uint Tell()
 795    {
 796       return position;
 797    }
 798
 799    bool Eof()
 800    {
 801       return eof;
 802    }
 803
 804    bool GetSize()
 805    {
 806       return totalSize;
 807    }
 808
 809    void Abort()
 810    {
 811       aborted = true;
 812    }
 813 private:
 814
 815    bool askedBody;
 816
 817    HTTPConnection connection;
 818    uint position;
 819    bool done;
 820    bool eof;
 821    int status;
 822    uint totalSize;
 823    bool chunked;
 824    bool close;
 825    uint chunkSize;
 826    char * relocation;
 827    String location;
 828
 829    // Buffering...
 830    byte buffer[HTTPFILE_BUFFERSIZE];
 831    uint bufferPos;
 832    uint bufferCount;
 833    bool aborted;
 834    bool totalSizeSet;
 835
 836    String contentType;
 837    String contentDisposition;
 838 }
 839
 840 public HTTPFile FileOpenURL(char * name)
 841 {
 842    HTTPFile f { };
 843    if(f.OpenURL(name, null, null))
 844       return f;
 845    else
 846    {
 847       delete f;
 848       return null;
 849    }
 850 }
 851
 852 #endif