ed2eb8f82c8d41ade5cadadb0dc2e9f276135a93
[sdk] / ecere / src / net / HTTPFile.ec
1 #ifndef ECERE_NONET
2
3 #include <stdio.h>
4
5 import "List"
6 import "network"
7 #ifndef ECERE_NOSSL
8 import "SSLSocket"
9 #endif
10
11 class ConnectionsHolder
12 {
13    List<HTTPConnection> connections { };
14
15    ~ConnectionsHolder()
16    {
17       HTTPConnection c;
18       while((c = connections[0]))
19          delete c;   // The HTTPConnection destructor will take out from the list
20    }
21 }
22
23 static ConnectionsHolder holder { };
24
25 namespace net;
26
27 /*static */define HTTPFILE_BUFFERSIZE = 65536;
28
29 static Mutex connectionsMutex { };
30
31 static class ServerNode : BTNode
32 {
33    char address[24];
34    bool resolved;
35    ~ServerNode()
36    {
37       char * name = (char *)key;
38       delete name;
39    }
40 }
41
42 static class ServerNameCache
43 {
44    BinaryTree servers
45    {
46       CompareKey = (void *)BinaryTree::CompareString;
47    };
48    Mutex mutex { };
49
50    ~ServerNameCache()
51    {
52       ServerNode server;
53       while((server = (ServerNode)servers.root))
54       {
55          servers.Remove(server);
56          delete server;
57       }
58    }
59    bool Resolve(char * host, char * address)
60    {
61       ServerNode server;
62       mutex.Wait();
63       server = (ServerNode)servers.FindString(host);
64       if(!server)
65       {
66          server = ServerNode { key = (uintptr)CopyString(host) };
67          servers.Add(server);
68          server.resolved = GetAddressFromName(host, server.address);
69       }
70       mutex.Release();
71       if(server.resolved)
72          strcpy(address, server.address);
73       return server.resolved;
74    }
75 };
76
77 static ServerNameCache serverNameCache { };
78
79 static char * GetString(char * string, char * what, int count)
80 {
81    int c;
82    for(c = 0; what[c]; c++)
83    {
84       if((count && c >= count) || (string[c] != what[c] && tolower(string[c]) != tolower(what[c])))
85          return null;
86    }
87    return string + c;
88 }
89
90 #ifdef ECERE_NOSSL
91 private class HTTPConnection : Socket
92 #else
93 private class HTTPConnection : SSLSocket
94 #endif
95 {
96    class_no_expansion;
97    char * server;
98    int port;
99    HTTPFile file;
100    bool secure;
101
102    processAlone = true;
103
104    ~HTTPConnection()
105    {
106       // printf("Before TakeOut we have %d connections:\n", holder.connections.count); for(c : holder.connections) ::PrintLn(c.server); ::PrintLn("");
107       holder.connections.TakeOut(this);
108       /*
109       PrintLn(server, " Connection Closed (", holder.connections.count, ") opened");
110       printf("Now we have %d connections:\n", holder.connections.count);
111       for(c : holder.connections)
112       {
113          ::PrintLn(c.server);
114       }
115       ::PrintLn("");
116       */
117       delete server;
118    }
119
120    void OnDisconnect(int code)
121    {
122       connectionsMutex.Wait();
123       if(file)
124          delete file.connection; // This decrements the file's reference
125
126       connectionsMutex.Release();
127
128       // PrintLn(server, " Disconnected Us");
129
130       delete this;         // The 'connections' reference
131    }
132
133    uint Open_OnReceive(const byte * buffer, uint count)
134    {
135       HTTPFile file = this.file;
136       int pos = 0;
137       int c;
138       while(!file.done)
139       {
140          bool gotEndLine = false;
141          for(c = 0; c<(int)count-1; c++)
142          {
143             if(buffer[c] == '\r' && buffer[c+1] == '\n')
144             {
145                gotEndLine = true;
146                break;
147             }
148          }
149          if(!gotEndLine)
150             // Incomplete packet
151             return pos;
152          if(c<count)
153          {
154             char * string = (char *)buffer;
155
156 #ifdef _DEBUG
157             fwrite(buffer, 1, c, stdout);
158             puts("");
159 #endif
160
161             if(!c)
162             {
163                //if(file.openStarted)
164                   file.done = true;
165             }
166             else
167             {
168                //file.openStarted = true;
169                if((string = GetString((char *)buffer, "HTTP/1.1 ", count)) ||
170                   (string = GetString((char *)buffer, "HTTP/1.0 ", count)))
171                {
172                   file.status = atoi(string);
173                }
174                else if((string = GetString((char *)buffer, "Transfer-Encoding: ", count)))
175                {
176                   if(!strnicmp(string, "chunked", strlen("chunked")))
177                   {
178                      file.chunked = true;
179                   }
180                }
181                else if((string = GetString((char *)buffer, "Content-Length: ", count)))
182                {
183                   file.totalSize = atoi(string);
184                   file.totalSizeSet = true;
185                }
186                else if((string = GetString((char *)buffer, "Content-Type: ", count)))
187                {
188                   char * cr = strstr(string, "\r");
189                   char * lf = strstr(string, "\n");
190                   int len;
191                   if(cr)
192                      len = cr - string;
193                   else if(lf)
194                      len = lf - string;
195                   else
196                      len = strlen(string);
197
198                   file.contentType = new char[len+1];
199                   memcpy(file.contentType, string, len);
200                   file.contentType[len] = 0;
201                }
202                else if((string = GetString((char *)buffer, "Content-disposition: ", count)))
203                {
204                   char * cr = strstr(string, "\r");
205                   char * lf = strstr(string, "\n");
206                   int len;
207                   if(cr)
208                      len = cr - string;
209                   else if(lf)
210                      len = lf - string;
211                   else
212                      len = strlen(string);
213
214                   file.contentDisposition = new char[len+1];
215                   memcpy(file.contentDisposition, string, len);
216                   file.contentDisposition[len] = 0;
217                }
218                else if((string = GetString((char *)buffer, "Connection: ", count)))
219                {
220                   if(!strnicmp(string, "close", strlen("close")))
221                   {
222                      file.close = true;
223                   }
224                }
225                else if((string = GetString((char *)buffer, "Location: ", count)))
226                {
227                   if(file.relocation)
228                   {
229                      strncpy(file.relocation, (char *)buffer + 10, c - 10);
230                      file.relocation[c - 10] = '\0';
231                   }
232                }
233             }
234             // return c+2;
235             if(buffer[c] == '\r') c++;
236             if(buffer[c] == '\n') c++;
237             pos += c;
238             count -= c;
239             buffer += c;
240
241          }
242          else
243             break;
244       }
245
246       return pos;
247    }
248
249    uint Read_OnReceive(const byte * buffer, uint count)
250    {
251       HTTPFile file = this.file;
252       if(file)
253       {
254          int read;
255          if(file.chunked && !file.chunkSize)
256          {
257             int pos = 0, c = 0;
258             char * string = null;
259             bool ready = false;
260             while(pos + c < (int)count-3)
261             {
262                if(buffer[pos+c] == '\r' && buffer[pos+c+1] == '\n')
263                {
264                   if(string)
265                   {
266                      ready = true;
267                      pos += c + 2;
268                      if(buffer[pos] == '\r' && buffer[pos+1] =='\n') pos+= 2;
269                      break;
270                   }
271                   else
272                   {
273                      pos += 2;
274                      c = 0;
275                   }
276                }
277                else
278                {
279                   if(!string)
280                      string = buffer + pos;
281                   c++;
282                }
283             }
284             if(ready)
285             {
286                file.chunkSize = strtol(string, null, 16);
287                if(!file.chunkSize)
288                {
289                   file.connection.file = null;
290                   if(file.close)
291                      file.connection.Disconnect(0);
292                   delete file.connection; // This decrements the file's reference
293                }
294             }
295             return pos;
296          }
297
298          read = Min(count, HTTPFILE_BUFFERSIZE - file.bufferCount);
299          if(file.chunked)
300          {
301             read = Min(read, file.chunkSize);
302             file.chunkSize -= read;
303          }
304          if(read)
305          {
306             memcpy(file.buffer + file.bufferCount, buffer, read);
307             file.bufferCount += read;
308          }
309          return read;
310       }
311       else
312          return count;
313    }
314 }
315
316 public class HTTPFile : File
317 {
318    bool reuseConnection;
319    reuseConnection = true;
320 public:
321    property bool reuseConnection
322    {
323       set { reuseConnection = value; }
324       get { return reuseConnection; }
325    }
326    property String contentType
327    {
328       get { return contentType; }
329    }
330    property String contentDisposition
331    {
332       get { return contentDisposition; }
333    }
334
335    bool OpenURL(char * name, char * referer, char * relocation)
336    {
337       return RetrieveHead(name, referer, relocation, false);
338    }
339
340 private:
341
342    bool RetrieveHead(char * name, char * referer, char * relocation, bool askBody)
343    {
344       bool result = false;
345       String http, https;
346       if(!this || !name) return false;
347       http = strstr(name, "http://");
348       if(http != name) http = null;
349       https = http ? null : strstr(name, "https://"); if(https && https != name) https = null;
350
351       askedBody = askBody;
352
353       done = false;
354       delete contentType;
355       delete contentDisposition;
356       // ::PrintLn("Opening ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID(), "\n");
357       if(this && (http || https))
358       {
359          char server[1024];
360          char msg[1024];
361          int len;
362          char * serverStart = http ? ecere::net::GetString(name, "http://", 0) : ecere::net::GetString(name, "https://", 0);
363          char * fileName = strstr(serverStart, "/");
364          int port = https ? 443 : 80;
365          char * colon;
366          bool reuse = false;
367          HTTPConnection connection = null;
368          close = false;
369
370          if(fileName)
371          {
372             fileName++;
373             memcpy(server, serverStart, fileName - serverStart - 1);
374             server[fileName - serverStart - 1] = '\0';
375          }
376          else
377             strcpy(server, serverStart);
378
379          if(relocation && !fileName && name[strlen(name)-1] != '/')
380          {
381             strcpy(relocation, http ? "http://" : "https://");
382             strcat(relocation, server);
383             strcat(relocation, "/");
384          }
385
386          colon = strchr(server, ':');
387          if(colon)
388          {
389             port = atoi(colon+1);
390             *colon = '\0';
391          }
392
393          connectionsMutex.Wait();
394          if(this.connection)
395          {
396             this.connection.file = null;
397             if(close)
398                this.connection.Disconnect(0);
399             delete this.connection;
400          }
401
402          if(chunked)
403          {
404             while(this.connection && this.connection.connected && this.connection.file)
405             {
406                connectionsMutex.Release();
407                this.connection.Process();
408                connectionsMutex.Wait();
409             }
410          }
411
412          if(reuseConnection)
413          {
414             bool retry = true;
415             while(retry)
416             {
417                retry = false;
418                connection = null;
419                for(c : holder.connections)
420                {
421                   if(!strcmpi(c.server, server) && c.port == port && c.secure == (https ? true : false))
422                   {
423                      if(!c.file && c.connected)
424                      {
425                         connection = c;      // TOFIX: 'incref c' doesn't work
426                         incref connection;      // HTTPFile reference if we keep it
427                         connectionsMutex.Release();
428                         connection.ProcessTimeOut(0.000001);
429                         connectionsMutex.Wait();
430                         if(!connection.connected || connection.file)
431                         {
432                            // We're disconnected or reused already...
433                            retry = true;
434                            delete connection;
435                         }
436                         break;
437                      }
438                   }
439                }
440             }
441             if(connection)
442             {
443                // ::PrintLn("Reusing Connection ", (uint64)connection, " for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID(), "\n");
444                reuse = true;
445                connection.file = this;
446             }
447          }
448
449       tryagain:
450          if(!connection)
451          {
452             char ipAddress[1024];
453             connection = HTTPConnection
454             {
455 #ifndef ECERE_NOSSL
456                autoEstablish = https ? true : false
457 #endif
458             };
459             incref connection;      // HTTPFile reference on success
460
461             connection.file = this;
462
463             connectionsMutex.Release();
464
465             if(serverNameCache.Resolve(server, ipAddress))
466             {
467                // ::PrintLn("No Connection - Connecting ", (uint64)connection, " for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID());
468                if(connection.Connect(ipAddress /*server*/, port))
469                {
470                   //::PrintLn("Successfully Connected for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID());
471                   //::PrintLn("Waiting on connectionsMutex for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID());
472                   connectionsMutex.Wait();
473
474                   connection.server = CopyString(server);
475                   connection.port = port;
476                   connection.secure = https ? true : false;
477
478                   //::PrintLn("Got connectionsMutex for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID());
479                   holder.connections.Add(connection);
480                   /*
481                   printf("Now we have %d connections:\n", holder.connections.count);
482                   for(c : holder.connections)
483                   {
484                      String s = c.server;
485                      ::Print("Server: ");
486                      ::PrintLn(c.server);
487                   }
488                   ::PrintLn("");
489                   */
490                   incref connection;   // Global List Reference
491                }
492                else
493                {
494                   // ::PrintLn("Connection Failed for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID());
495                   connectionsMutex.Wait();
496                   delete connection;
497                }
498             }
499             else
500             {
501                connectionsMutex.Wait();
502                delete connection;
503             }
504          }
505
506          if(connection)
507          {
508             incref connection;      // local reference
509
510             connection.OnReceive = HTTPConnection::Open_OnReceive;
511             connection.file = this;
512             this.connection = connection;
513             this.relocation = relocation;
514             //openStarted = false;
515
516             totalSizeSet = false;   // HEAD will sometimes give you 0!
517             strcpy(msg, askBody ? "GET /" : "HEAD /");
518
519             if(fileName)
520             {
521                byte ch;
522                int c;
523                len = strlen(msg);
524                for(c = 0; (ch = fileName[c]); c++)
525                {
526                   if(ch <= 32 || ch > 128)
527                   {
528                      byte nibble;
529                      msg[len++] = '%';
530                      nibble = (ch & 0xF0) >> 4;
531                      msg[len++] = (nibble > 9) ? (nibble - 10 + 'a') : (nibble + '0');
532                      nibble = ch & 0x0F;
533                      msg[len++] = (nibble > 9) ? (nibble - 10 + 'a') : (nibble + '0');
534                   }
535                   else
536                      msg[len++] = ch;
537                }
538                msg[len] = '\0';
539             }
540
541             strcat(msg, " HTTP/1.1\r\nHost: ");
542             //strcat(msg, " HTTP/1.0\r\nHost: ");
543             strcat(msg, server);
544             strcat(msg, "\r\n");
545             strcat(msg, "Accept-Charset: UTF-8\r\n");
546             //strcat(msg, "Accept-Charset: ISO-8859-1\r\n");
547             strcat(msg, "Connection: Keep-Alive\r\n");
548             if(referer)
549             {
550                strcat(msg, "Referer: ");
551                strcat(msg, referer);
552                strcat(msg, "\r\n");
553             }
554             strcat(msg, "\r\n");
555             len = strlen(msg);
556
557             //::PrintLn("Releasing connectionsMutex before GET for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID());
558             connectionsMutex.Release();
559
560             // ::PrintLn("Sending GET for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID(), "\n");
561             connection.Send(msg, len);
562
563             while(this.connection && this.connection.connected && !done)
564             {
565                this.connection.Process();
566             }
567             //::PrintLn("Got DONE for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID(), "\n");
568
569             if(this.connection)
570             {
571                if(name != location)
572                {
573                   delete location;
574                   location = CopyString(name);
575                }
576                if(status == 200 || (!status && totalSizeSet))
577                {
578                   if(askBody)
579                      this.connection.OnReceive = HTTPConnection::Read_OnReceive;
580
581                   result = true;
582                   connectionsMutex.Wait();
583                }
584                else
585                {
586                   if(askBody)
587                   {
588                      if(chunked)
589                      {
590                         //bool wait = false;
591                         this.connection.OnReceive = HTTPConnection::Read_OnReceive;
592                         while(!eof)
593                         {
594                            if(!this.connection)
595                               eof = true;
596                            else
597                            {
598                               // First time check if we already have bytes, second time wait for an event
599                               this.connection.Process();
600                               //wait = true;
601                            }
602                         }
603                      }
604                      else if(totalSizeSet)
605                      {
606                         // Is it possible to have Content-Length set but not chunked?
607                         done = false;
608                         this.connection.OnReceive = HTTPConnection::Read_OnReceive;
609                         while(this.connection && this.connection.connected && position + (bufferCount - bufferPos) < totalSize)
610                         {
611                            connection.Process();
612                            position += bufferCount - bufferPos;
613                            bufferCount = 0;
614                            bufferPos = 0;
615                         }
616                      }
617                   }
618
619                   connectionsMutex.Wait();
620                   if(this.connection)
621                   {
622                      this.connection.OnReceive = null;
623                      this.connection.file = null;
624
625                      if(close)
626                      {
627                         this.connection.Disconnect(0);
628                         connection = null;
629                      }
630                   }
631
632                   status = 0;
633                   delete this.connection; // This decrements the file's reference
634                   this.relocation = null;
635                   totalSize = 0;
636                   totalSizeSet = false;
637                   done = false;
638                   eof = false;
639                   position = 0;
640                   bufferPos = 0;
641                   bufferCount = 0;
642                   chunked = false;
643                }
644             }
645             else
646             {
647                connectionsMutex.Wait();
648             }
649             if(reuse && !status && connection && !connection.connected)
650             {
651                delete connection;
652                reuse = false;
653                goto tryagain;
654             }
655             else
656                delete connection;
657          }
658          connectionsMutex.Release();
659       }
660       return result;
661    }
662
663    ~HTTPFile()
664    {
665       delete location;
666       delete contentType;
667       delete contentDisposition;
668       {
669          connectionsMutex.Wait();
670          if(connection)
671          {
672             if(totalSizeSet && askedBody)
673             {
674                done = false;
675                this.connection.OnReceive = HTTPConnection::Read_OnReceive;
676                while(this.connection && this.connection.connected && position + (bufferCount - bufferPos) < totalSize)
677                {
678                   connectionsMutex.Release();
679                   connection.Process();
680                   connectionsMutex.Wait();
681                   position += bufferCount - bufferPos;
682                   bufferCount = 0;
683                   bufferPos = 0;
684                }
685                position = 0;
686             }
687
688             if(connection)
689             {
690                connection.file = null;
691                if(close)
692                   connection.Disconnect(0);
693                delete connection;
694             }
695          }
696          connectionsMutex.Release();
697
698          if(chunked)
699          {
700             while(connection && connection.connected && connection.file)
701             {
702                connectionsMutex.Release();
703                connection.Process();
704                connectionsMutex.Wait();
705             }
706          }
707          //::PrintLn("Done with ", (uint64)this);
708       }
709    }
710
711    int Read(byte * buffer, uint size, uint count)
712    {
713       uint readSize = size * count;
714       uint read = 0;
715       bool wait = false;
716       Time lastTime = GetTime();
717
718       if(!askedBody)
719       {
720          askedBody = true;
721          if(!RetrieveHead(this.location, null, null, true))
722             return 0;
723       }
724
725       if(totalSizeSet && position >= totalSize)
726          eof = true;
727       while(!eof && read < readSize && !aborted)
728       {
729          uint numbytes = bufferCount - bufferPos;
730          numbytes = Min(numbytes, readSize - read);
731          if(totalSizeSet)
732             numbytes = Min(numbytes, totalSize - position);
733
734          if(numbytes)
735          {
736             lastTime = GetTime();
737             memcpy(buffer + read, this.buffer + bufferPos, numbytes);
738             bufferPos += numbytes;
739             position += numbytes;
740             read += numbytes;
741
742             lastTime = GetTime();
743
744             if(bufferPos > HTTPFILE_BUFFERSIZE / 2)
745             {
746                // Shift bytes back to beginning of buffer
747                uint shift = bufferCount - bufferPos;
748                if(shift)
749                   memmove(this.buffer, this.buffer + bufferPos, shift);
750                bufferCount -= bufferPos;
751                bufferPos = 0;
752             }
753          }
754          else
755          {
756             if(!connection || !connection.connected)
757                eof = true;
758             else
759             {
760                // First time check if we already have bytes, second time wait for an event
761                connection.Process();
762                if(wait && bufferCount - bufferPos == 0 && GetTime() - lastTime > 5)
763                   eof = true;
764                wait = true;
765             }
766          }
767          if(totalSizeSet && position >= totalSize)
768             eof = true;
769       }
770       return read / size;
771    }
772
773    int Write(byte * buffer, uint size, uint count)
774    {
775       return 0;
776    }
777
778    bool Getc(char * ch)
779    {
780       int read = Read(ch, 1, 1);
781       return !eof && read != 0;
782    }
783
784    bool Putc(char ch)
785    {
786       return false;
787    }
788
789    bool Puts(char * string)
790    {
791       return false;
792    }
793
794    bool Seek(int pos, FileSeekMode mode)
795    {
796       if(mode == start && bufferPos == 0 && pos <= bufferCount && pos >= 0)
797       {
798          bufferPos = pos;
799          return true;
800       }
801       else if(mode == current && bufferPos == 0 && (position + pos) <= bufferCount && (position + pos) >= 0)
802       {
803          bufferPos = position + pos;
804          return true;
805       }
806       else if(mode == end && totalSizeSet && bufferPos == 0 && bufferCount == totalSize && (totalSize - pos) <= bufferCount && (totalSize - pos) >= 0)
807       {
808          bufferPos = totalSize - pos;
809          return true;
810       }
811       return false;
812    }
813
814    uint Tell()
815    {
816       return position;
817    }
818
819    bool Eof()
820    {
821       return eof;
822    }
823
824    uint GetSize()
825    {
826       return totalSize;
827    }
828
829    void Abort()
830    {
831       aborted = true;
832    }
833 private:
834
835    bool askedBody;
836
837    HTTPConnection connection;
838    uint position;
839    bool done;
840    bool eof;
841    int status;
842    uint totalSize;
843    bool chunked;
844    bool close;
845    uint chunkSize;
846    char * relocation;
847    String location;
848
849    // Buffering...
850    byte buffer[HTTPFILE_BUFFERSIZE];
851    uint bufferPos;
852    uint bufferCount;
853    bool aborted;
854    bool totalSizeSet;
855
856    String contentType;
857    String contentDisposition;
858 }
859
860 public HTTPFile FileOpenURL(char * name)
861 {
862    HTTPFile f { };
863    if(f.OpenURL(name, null, null))
864       return f;
865    else
866    {
867       delete f;
868       return null;
869    }
870 }
871
872 #endif