The Ecere SDK - sdk/blob - ecere/src/net/HTTPFile.ec

   1 #ifndef ECERE_NONET
   2
   3 #include <stdio.h>
   4
   5 import "List"
   6 import "network"
   7 #ifndef ECERE_NOSSL
   8 import "SSLSocket"
   9 #endif
  10
  11 class ConnectionsHolder
  12 {
  13    List<HTTPConnection> connections { };
  14
  15    ~ConnectionsHolder()
  16    {
  17       HTTPConnection c;
  18       while((c = connections[0]))
  19          delete c;   // The HTTPConnection destructor will take out from the list
  20    }
  21 }
  22
  23 static ConnectionsHolder holder { };
  24
  25 namespace net;
  26
  27 /*static */define HTTPFILE_BUFFERSIZE = 65536;
  28
  29 static Mutex connectionsMutex { };
  30
  31 static class ServerNode : BTNode
  32 {
  33    char address[24];
  34    bool resolved;
  35    ~ServerNode()
  36    {
  37       char * name = (char *)key;
  38       delete name;
  39    }
  40 }
  41
  42 static class ServerNameCache
  43 {
  44    BinaryTree servers
  45    {
  46       CompareKey = (void *)BinaryTree::CompareString;
  47    };
  48    Mutex mutex { };
  49
  50    ~ServerNameCache()
  51    {
  52       ServerNode server;
  53       while((server = (ServerNode)servers.root))
  54       {
  55          servers.Remove(server);
  56          delete server;
  57       }
  58    }
  59    bool Resolve(char * host, char * address)
  60    {
  61       ServerNode server;
  62       mutex.Wait();
  63       server = (ServerNode)servers.FindString(host);
  64       if(!server)
  65       {
  66          server = ServerNode { key = (uintptr)CopyString(host) };
  67          servers.Add(server);
  68          server.resolved = GetAddressFromName(host, server.address);
  69       }
  70       mutex.Release();
  71       if(server.resolved)
  72          strcpy(address, server.address);
  73       return server.resolved;
  74    }
  75 };
  76
  77 static ServerNameCache serverNameCache { };
  78
  79 static char * GetString(char * string, char * what, int count)
  80 {
  81    int c;
  82    for(c = 0; what[c]; c++)
  83    {
  84       if((count && c >= count) || (string[c] != what[c] && tolower(string[c]) != tolower(what[c])))
  85          return null;
  86    }
  87    return string + c;
  88 }
  89
  90 #ifdef ECERE_NOSSL
  91 private class HTTPConnection : Socket
  92 #else
  93 private class HTTPConnection : SSLSocket
  94 #endif
  95 {
  96    class_no_expansion;
  97    char * server;
  98    int port;
  99    HTTPFile file;
 100    bool secure;
 101
 102    processAlone = true;
 103
 104    ~HTTPConnection()
 105    {
 106       // printf("Before TakeOut we have %d connections:\n", holder.connections.count); for(c : holder.connections) ::PrintLn(c.server); ::PrintLn("");
 107       holder.connections.TakeOut(this);
 108       /*
 109       PrintLn(server, " Connection Closed (", holder.connections.count, ") opened");
 110       printf("Now we have %d connections:\n", holder.connections.count);
 111       for(c : holder.connections)
 112       {
 113          ::PrintLn(c.server);
 114       }
 115       ::PrintLn("");
 116       */
 117       delete server;
 118    }
 119
 120    void OnDisconnect(int code)
 121    {
 122       connectionsMutex.Wait();
 123       if(file)
 124          delete file.connection; // This decrements the file's reference
 125
 126       connectionsMutex.Release();
 127
 128       // PrintLn(server, " Disconnected Us");
 129
 130       delete this;         // The 'connections' reference
 131    }
 132
 133    uint Open_OnReceive(const byte * buffer, uint count)
 134    {
 135       HTTPFile file = this.file;
 136       int pos = 0;
 137       int c;
 138       while(!file.done)
 139       {
 140          bool gotEndLine = false;
 141          for(c = 0; c<(int)count-1; c++)
 142          {
 143             if(buffer[c] == '\r' && buffer[c+1] == '\n')
 144             {
 145                gotEndLine = true;
 146                break;
 147             }
 148          }
 149          if(!gotEndLine)
 150             // Incomplete packet
 151             return pos;
 152          if(c<count)
 153          {
 154             char * string = (char *)buffer;
 155
 156 #ifdef _DEBUG
 157             fwrite(buffer, 1, c, stdout);
 158             puts("");
 159 #endif
 160
 161             if(!c)
 162             {
 163                //if(file.openStarted)
 164                   file.done = true;
 165             }
 166             else
 167             {
 168                //file.openStarted = true;
 169                if((string = GetString((char *)buffer, "HTTP/1.1 ", count)) ||
 170                   (string = GetString((char *)buffer, "HTTP/1.0 ", count)))
 171                {
 172                   file.status = atoi(string);
 173                }
 174                else if(string = GetString((char *)buffer, "Transfer-Encoding: ", count))
 175                {
 176                   if(!strnicmp(string, "chunked", strlen("chunked")))
 177                   {
 178                      file.chunked = true;
 179                   }
 180                }
 181                else if(string = GetString((char *)buffer, "Content-Length: ", count))
 182                {
 183                   file.totalSize = atoi(string);
 184                   file.totalSizeSet = true;
 185                }
 186                else if(string = GetString((char *)buffer, "Content-Type: ", count))
 187                {
 188                   char * cr = strstr(string, "\r");
 189                   char * lf = strstr(string, "\n");
 190                   int len;
 191                   if(cr)
 192                      len = cr - string;
 193                   else if(lf)
 194                      len = lf - string;
 195                   else
 196                      len = strlen(string);
 197
 198                   file.contentType = new char[len+1];
 199                   memcpy(file.contentType, string, len);
 200                   file.contentType[len] = 0;
 201                }
 202                else if(string = GetString((char *)buffer, "Content-disposition: ", count))
 203                {
 204                   char * cr = strstr(string, "\r");
 205                   char * lf = strstr(string, "\n");
 206                   int len;
 207                   if(cr)
 208                      len = cr - string;
 209                   else if(lf)
 210                      len = lf - string;
 211                   else
 212                      len = strlen(string);
 213
 214                   file.contentDisposition = new char[len+1];
 215                   memcpy(file.contentDisposition, string, len);
 216                   file.contentDisposition[len] = 0;
 217                }
 218                else if(string = GetString((char *)buffer, "Connection: ", count))
 219                {
 220                   if(!strnicmp(string, "close", strlen("close")))
 221                   {
 222                      file.close = true;
 223                   }
 224                }
 225                else if(string = GetString((char *)buffer, "Location: ", count))
 226                {
 227                   if(file.relocation)
 228                   {
 229                      strncpy(file.relocation, buffer + 10, c - 10);
 230                      file.relocation[c - 10] = '\0';
 231                   }
 232                }
 233             }
 234             // return c+2;
 235             if(buffer[c] == '\r') c++;
 236             if(buffer[c] == '\n') c++;
 237             pos += c;
 238             count -= c;
 239             buffer += c;
 240
 241          }
 242          else
 243             break;
 244       }
 245
 246       return pos;
 247    }
 248
 249    uint Read_OnReceive(const byte * buffer, uint count)
 250    {
 251       HTTPFile file = this.file;
 252       if(file)
 253       {
 254          int read;
 255          if(file.chunked && !file.chunkSize)
 256          {
 257             int pos = 0, c = 0;
 258             char * string = null;
 259             bool ready = false;
 260             while(pos + c < (int)count-3)
 261             {
 262                if(buffer[pos+c] == '\r' && buffer[pos+c+1] == '\n')
 263                {
 264                   if(string)
 265                   {
 266                      ready = true;
 267                      pos += c + 2;
 268                      if(buffer[pos] == '\r' && buffer[pos+1] =='\n') pos+= 2;
 269                      break;
 270                   }
 271                   else
 272                   {
 273                      pos += 2;
 274                      c = 0;
 275                   }
 276                }
 277                else
 278                {
 279                   if(!string)
 280                      string = buffer + pos;
 281                   c++;
 282                }
 283             }
 284             if(ready)
 285             {
 286                file.chunkSize = strtol(string, null, 16);
 287                if(!file.chunkSize)
 288                {
 289                   file.connection.file = null;
 290                   delete file.connection; // This decrements the file's reference
 291                }
 292             }
 293             return pos;
 294          }
 295
 296          read = Min(count, HTTPFILE_BUFFERSIZE - file.bufferCount);
 297          if(file.chunked)
 298          {
 299             read = Min(read, file.chunkSize);
 300             file.chunkSize -= read;
 301          }
 302          if(read)
 303          {
 304             memcpy(file.buffer + file.bufferCount, buffer, read);
 305             file.bufferCount += read;
 306          }
 307          return read;
 308       }
 309       else
 310          return count;
 311    }
 312 }
 313
 314 public class HTTPFile : File
 315 {
 316    bool reuseConnection;
 317    reuseConnection = true;
 318 public:
 319    property bool reuseConnection
 320    {
 321       set { reuseConnection = value; }
 322       get { return reuseConnection; }
 323    }
 324    property String contentType
 325    {
 326       get { return contentType; }
 327    }
 328    property String contentDisposition
 329    {
 330       get { return contentDisposition; }
 331    }
 332
 333    bool OpenURL(char * name, char * referer, char * relocation)
 334    {
 335       return RetrieveHead(name, referer, relocation, false);
 336    }
 337
 338 private:
 339
 340    bool RetrieveHead(char * name, char * referer, char * relocation, bool askBody)
 341    {
 342       bool result = false;
 343       String http, https;
 344       if(!this || !name) return false;
 345       http = strstr(name, "http://");
 346       if(http != name) http = null;
 347       https = http ? null : strstr(name, "https://"); if(https && https != name) https = null;
 348
 349       askedBody = askBody;
 350
 351       done = false;
 352       delete contentType;
 353       delete contentDisposition;
 354       // ::PrintLn("Opening ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID(), "\n");
 355       if(this && (http || https))
 356       {
 357          char server[1024];
 358          char msg[1024];
 359          int len;
 360          char * serverStart = http ? ecere::net::GetString(name, "http://", 0) : ecere::net::GetString(name, "https://", 0);
 361          char * fileName = strstr(serverStart, "/");
 362          int port = https ? 443 : 80;
 363          char * colon;
 364          bool reuse = false;
 365          HTTPConnection connection = null;
 366          close = false;
 367
 368          if(fileName)
 369          {
 370             fileName++;
 371             memcpy(server, serverStart, fileName - serverStart - 1);
 372             server[fileName - serverStart - 1] = '\0';
 373          }
 374          else
 375             strcpy(server, serverStart);
 376
 377          if(relocation && !fileName && name[strlen(name)-1] != '/')
 378          {
 379             strcpy(relocation, http ? "http://" : "https://");
 380             strcat(relocation, server);
 381             strcat(relocation, "/");
 382          }
 383
 384          colon = strchr(server, ':');
 385          if(colon)
 386          {
 387             port = atoi(colon+1);
 388             *colon = '\0';
 389          }
 390
 391          connectionsMutex.Wait();
 392          if(this.connection)
 393          {
 394             this.connection.file = null;
 395             if(close)
 396                this.connection.Disconnect(0);
 397             delete this.connection;
 398          }
 399
 400          if(chunked)
 401          {
 402             while(this.connection && this.connection.connected && this.connection.file)
 403             {
 404                connectionsMutex.Release();
 405                this.connection.Process();
 406                connectionsMutex.Wait();
 407             }
 408          }
 409
 410          if(reuseConnection)
 411          {
 412             bool retry = true;
 413             while(retry)
 414             {
 415                retry = false;
 416                connection = null;
 417                for(c : holder.connections)
 418                {
 419                   if(!strcmpi(c.server, server) && c.port == port && c.secure == (https ? true : false))
 420                   {
 421                      if(!c.file && c.connected)
 422                      {
 423                         connection = c;      // TOFIX: 'incref c' doesn't work
 424                         incref connection;      // HTTPFile reference if we keep it
 425                         connectionsMutex.Release();
 426                         connection.ProcessTimeOut(0.000001);
 427                         connectionsMutex.Wait();
 428                         if(!connection.connected || connection.file)
 429                         {
 430                            // We're disconnected or reused already...
 431                            retry = true;
 432                            delete connection;
 433                         }
 434                         break;
 435                      }
 436                   }
 437                }
 438             }
 439             if(connection)
 440             {
 441                // ::PrintLn("Reusing Connection ", (uint64)connection, " for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID(), "\n");
 442                reuse = true;
 443                connection.file = this;
 444             }
 445          }
 446
 447       tryagain:
 448          if(!connection)
 449          {
 450             char ipAddress[1024];
 451             connection = HTTPConnection
 452             {
 453 #ifndef ECERE_NOSSL
 454                autoEstablish = https ? true : false
 455 #endif
 456             };
 457             incref connection;      // HTTPFile reference on success
 458
 459             connection.file = this;
 460
 461             connectionsMutex.Release();
 462
 463             if(serverNameCache.Resolve(server, ipAddress))
 464             {
 465                // ::PrintLn("No Connection - Connecting ", (uint64)connection, " for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID());
 466                if(connection.Connect(ipAddress /*server*/, port))
 467                {
 468                   //::PrintLn("Successfully Connected for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID());
 469                   //::PrintLn("Waiting on connectionsMutex for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID());
 470                   connectionsMutex.Wait();
 471
 472                   connection.server = CopyString(server);
 473                   connection.port = port;
 474                   connection.secure = https ? true : false;
 475
 476                   //::PrintLn("Got connectionsMutex for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID());
 477                   holder.connections.Add(connection);
 478                   /*
 479                   printf("Now we have %d connections:\n", holder.connections.count);
 480                   for(c : holder.connections)
 481                   {
 482                      String s = c.server;
 483                      ::Print("Server: ");
 484                      ::PrintLn(c.server);
 485                   }
 486                   ::PrintLn("");
 487                   */
 488                   incref connection;   // Global List Reference
 489                }
 490                else
 491                {
 492                   // ::PrintLn("Connection Failed for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID());
 493                   connectionsMutex.Wait();
 494                   delete connection;
 495                }
 496             }
 497             else
 498             {
 499                connectionsMutex.Wait();
 500                delete connection;
 501             }
 502          }
 503
 504          if(connection)
 505          {
 506             incref connection;      // local reference
 507
 508             connection.OnReceive = HTTPConnection::Open_OnReceive;
 509             connection.file = this;
 510             this.connection = connection;
 511             this.relocation = relocation;
 512             //openStarted = false;
 513
 514             totalSizeSet = false;   // HEAD will sometimes give you 0!
 515             strcpy(msg, askBody ? "GET /" : "HEAD /");
 516
 517             if(fileName)
 518             {
 519                byte ch;
 520                int c;
 521                len = strlen(msg);
 522                for(c = 0; (ch = fileName[c]); c++)
 523                {
 524                   if(ch <= 32 || ch > 128)
 525                   {
 526                      byte nibble;
 527                      msg[len++] = '%';
 528                      nibble = (ch & 0xF0) >> 4;
 529                      msg[len++] = (nibble > 9) ? (nibble - 10 + 'a') : (nibble + '0');
 530                      nibble = ch & 0x0F;
 531                      msg[len++] = (nibble > 9) ? (nibble - 10 + 'a') : (nibble + '0');
 532                   }
 533                   else
 534                      msg[len++] = ch;
 535                }
 536                msg[len] = '\0';
 537             }
 538
 539             strcat(msg, " HTTP/1.1\r\nHost: ");
 540             //strcat(msg, " HTTP/1.0\r\nHost: ");
 541             strcat(msg, server);
 542             strcat(msg, "\r\n");
 543             strcat(msg, "Accept-Charset: UTF-8\r\n");
 544             //strcat(msg, "Accept-Charset: ISO-8859-1\r\n");
 545             strcat(msg, "Connection: Keep-Alive\r\n");
 546             if(referer)
 547             {
 548                strcat(msg, "Referer: ");
 549                strcat(msg, referer);
 550                strcat(msg, "\r\n");
 551             }
 552             strcat(msg, "\r\n");
 553             len = strlen(msg);
 554
 555             //::PrintLn("Releasing connectionsMutex before GET for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID());
 556             connectionsMutex.Release();
 557
 558             // ::PrintLn("Sending GET for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID(), "\n");
 559             connection.Send(msg, len);
 560
 561             while(this.connection && this.connection.connected && !done)
 562             {
 563                this.connection.Process();
 564             }
 565             //::PrintLn("Got DONE for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID(), "\n");
 566
 567             if(this.connection)
 568             {
 569                if(name != location)
 570                {
 571                   delete location;
 572                   location = CopyString(name);
 573                }
 574                if(status == 200 || (!status && totalSizeSet))
 575                {
 576                   if(askBody)
 577                      this.connection.OnReceive = HTTPConnection::Read_OnReceive;
 578
 579                   result = true;
 580                   connectionsMutex.Wait();
 581                }
 582                else
 583                {
 584                   if(askBody)
 585                   {
 586                      if(chunked)
 587                      {
 588                         bool wait = false;
 589                         this.connection.OnReceive = HTTPConnection::Read_OnReceive;
 590                         while(!eof)
 591                         {
 592                            if(!this.connection)
 593                               eof = true;
 594                            else
 595                            {
 596                               // First time check if we already have bytes, second time wait for an event
 597                               this.connection.Process();
 598                               wait = true;
 599                            }
 600                         }
 601                      }
 602                      else if(totalSizeSet)
 603                      {
 604                         done = false;
 605                         this.connection.OnReceive = HTTPConnection::Read_OnReceive;
 606                         while(this.connection && this.connection.connected && position < totalSize)
 607                         {
 608                            connection.Process();
 609                            position += bufferCount;
 610                            bufferCount = 0;
 611                         }
 612                      }
 613                   }
 614
 615                   connectionsMutex.Wait();
 616                   if(this.connection)
 617                   {
 618                      this.connection.OnReceive = null;
 619                      this.connection.file = null;
 620
 621                      if(close)
 622                      {
 623                         this.connection.Disconnect(0);
 624                         connection = null;
 625                      }
 626                   }
 627
 628                   status = 0;
 629                   delete this.connection; // This decrements the file's reference
 630                   this.relocation = null;
 631                   totalSize = 0;
 632                   totalSizeSet = false;
 633                   done = false;
 634                   eof = false;
 635                   position = 0;
 636                   bufferPos = 0;
 637                   bufferCount = 0;
 638                   chunked = false;
 639                }
 640             }
 641             else
 642             {
 643                connectionsMutex.Wait();
 644             }
 645             if(reuse && !status && connection && !connection.connected)
 646             {
 647                delete connection;
 648                reuse = false;
 649                goto tryagain;
 650             }
 651             else
 652                delete connection;
 653          }
 654          connectionsMutex.Release();
 655       }
 656       return result;
 657    }
 658
 659    ~HTTPFile()
 660    {
 661       delete location;
 662       delete contentType;
 663       delete contentDisposition;
 664       {
 665          connectionsMutex.Wait();
 666          if(connection)
 667          {
 668             if(totalSizeSet && askedBody)
 669             {
 670                done = false;
 671                this.connection.OnReceive = HTTPConnection::Read_OnReceive;
 672                while(this.connection && this.connection.connected && position + (bufferCount - bufferPos) < totalSize)
 673                {
 674                   connectionsMutex.Release();
 675                   connection.Process();
 676                   connectionsMutex.Wait();
 677                   position += bufferCount;
 678                   bufferCount = 0;
 679                }
 680                position = 0;
 681             }
 682
 683             if(connection)
 684             {
 685                connection.file = null;
 686                if(close)
 687                   connection.Disconnect(0);
 688                delete connection;
 689             }
 690          }
 691          connectionsMutex.Release();
 692
 693          if(chunked)
 694          {
 695             while(connection && connection.connected && connection.file)
 696             {
 697                connectionsMutex.Release();
 698                connection.Process();
 699                connectionsMutex.Wait();
 700             }
 701          }
 702          //::PrintLn("Done with ", (uint64)this);
 703       }
 704    }
 705
 706    int Read(byte * buffer, uint size, uint count)
 707    {
 708       uint readSize = size * count;
 709       uint read = 0;
 710       bool wait = false;
 711       Time lastTime = GetTime();
 712
 713       if(!askedBody)
 714       {
 715          askedBody = true;
 716          if(!RetrieveHead(this.location, null, null, true))
 717             return 0;
 718       }
 719
 720       if(totalSizeSet && position >= totalSize)
 721          eof = true;
 722       while(!eof && read < readSize && !aborted)
 723       {
 724          uint numbytes = bufferCount - bufferPos;
 725          numbytes = Min(numbytes, readSize - read);
 726          if(totalSizeSet)
 727             numbytes = Min(numbytes, totalSize - position);
 728
 729          if(numbytes)
 730          {
 731             lastTime = GetTime();
 732             memcpy(buffer + read, this.buffer + bufferPos, numbytes);
 733             bufferPos += numbytes;
 734             position += numbytes;
 735             read += numbytes;
 736
 737             lastTime = GetTime();
 738
 739             if(bufferPos > HTTPFILE_BUFFERSIZE / 2)
 740             {
 741                // Shift bytes back to beginning of buffer
 742                uint shift = bufferCount - bufferPos;
 743                if(shift)
 744                   memcpy(this.buffer, this.buffer + bufferPos, shift);
 745                bufferCount -= bufferPos;
 746                bufferPos = 0;
 747             }
 748          }
 749          else
 750          {
 751             if(!connection || !connection.connected)
 752                eof = true;
 753             else
 754             {
 755                // First time check if we already have bytes, second time wait for an event
 756                connection.Process();
 757                if(wait && bufferCount - bufferPos == 0 && GetTime() - lastTime > 5)
 758                   eof = true;
 759                wait = true;
 760             }
 761          }
 762          if(totalSizeSet && position >= totalSize)
 763             eof = true;
 764       }
 765       return read / size;
 766    }
 767
 768    int Write(byte * buffer, uint size, uint count)
 769    {
 770       return 0;
 771    }
 772
 773    bool Getc(char * ch)
 774    {
 775       int read = Read(ch, 1, 1);
 776       return !eof && read != 0;
 777    }
 778
 779    bool Putc(char ch)
 780    {
 781       return false;
 782    }
 783
 784    bool Puts(char * string)
 785    {
 786       return false;
 787    }
 788
 789    bool Seek(int pos, FileSeekMode mode)
 790    {
 791       if(mode == start && bufferPos == 0 && pos <= bufferCount & pos >= 0)
 792       {
 793          bufferPos = pos;
 794          return true;
 795       }
 796       else if(mode == current && bufferPos == 0 && (position + pos) <= bufferCount & (position + pos) >= 0)
 797       {
 798          bufferPos = position + pos;
 799          return true;
 800       }
 801       else if(mode == end && totalSizeSet && bufferPos == 0 && bufferCount == totalSize && (totalSize - pos) <= bufferCount & (totalSize - pos) >= 0)
 802       {
 803          bufferPos = totalSize - pos;
 804          return true;
 805       }
 806       return false;
 807    }
 808
 809    uint Tell()
 810    {
 811       return position;
 812    }
 813
 814    bool Eof()
 815    {
 816       return eof;
 817    }
 818
 819    bool GetSize()
 820    {
 821       return totalSize;
 822    }
 823
 824    void Abort()
 825    {
 826       aborted = true;
 827    }
 828 private:
 829
 830    bool askedBody;
 831
 832    HTTPConnection connection;
 833    uint position;
 834    bool done;
 835    bool eof;
 836    int status;
 837    uint totalSize;
 838    bool chunked;
 839    bool close;
 840    uint chunkSize;
 841    char * relocation;
 842    String location;
 843
 844    // Buffering...
 845    byte buffer[HTTPFILE_BUFFERSIZE];
 846    uint bufferPos;
 847    uint bufferCount;
 848    bool aborted;
 849    bool totalSizeSet;
 850
 851    String contentType;
 852    String contentDisposition;
 853 }
 854
 855 public HTTPFile FileOpenURL(char * name)
 856 {
 857    HTTPFile f { };
 858    if(f.OpenURL(name, null, null))
 859       return f;
 860    else
 861    {
 862       delete f;
 863       return null;
 864    }
 865 }
 866
 867 #endif