ecere; ide; compiler: Fixed some Clang warnings
[sdk] / ecere / src / net / HTTPFile.ec
1 #ifndef ECERE_NONET
2
3 #include <stdio.h>
4
5 import "List"
6 import "network"
7 #ifndef ECERE_NOSSL
8 import "SSLSocket"
9 #endif
10
11 class ConnectionsHolder
12 {
13    List<HTTPConnection> connections { };
14
15    ~ConnectionsHolder()
16    {
17       HTTPConnection c;
18       while((c = connections[0]))
19          delete c;   // The HTTPConnection destructor will take out from the list
20    }
21 }
22
23 static ConnectionsHolder holder { };
24
25 namespace net;
26
27 /*static */define HTTPFILE_BUFFERSIZE = 65536;
28
29 static Mutex connectionsMutex { };
30
31 static class ServerNode : BTNode
32 {
33    char address[24];
34    bool resolved;
35    ~ServerNode()
36    {
37       delete (char *)key;
38    }
39 }
40
41 static class ServerNameCache
42 {
43    BinaryTree servers
44    {
45       CompareKey = (void *)BinaryTree::CompareString;
46    };
47    Mutex mutex { };
48
49    ~ServerNameCache()
50    {
51       ServerNode server;
52       while((server = (ServerNode)servers.root))
53       {
54          servers.Remove(server);
55          delete server;
56       }
57    }
58    bool Resolve(const char * host, char * address)
59    {
60       ServerNode server;
61       mutex.Wait();
62       server = (ServerNode)servers.FindString(host);
63       if(!server)
64       {
65          server = ServerNode { key = (uintptr)CopyString(host) };
66          servers.Add(server);
67          server.resolved = GetAddressFromName(host, server.address);
68       }
69       mutex.Release();
70       if(server.resolved)
71          strcpy(address, server.address);
72       return server.resolved;
73    }
74 };
75
76 static ServerNameCache serverNameCache { };
77
78 static const char * GetString(const char * string, const char * what, int count)
79 {
80    int c;
81    for(c = 0; what[c]; c++)
82    {
83       if((count && c >= count) || (string[c] != what[c] && tolower(string[c]) != tolower(what[c])))
84          return null;
85    }
86    return string + c;
87 }
88
89 #ifdef ECERE_NOSSL
90 private class HTTPConnection : Socket
91 #else
92 private class HTTPConnection : SSLSocket
93 #endif
94 {
95    class_no_expansion;
96    char * server;
97    int port;
98    HTTPFile file;
99    bool secure;
100
101    processAlone = true;
102
103    ~HTTPConnection()
104    {
105       // printf("Before TakeOut we have %d connections:\n", holder.connections.count); for(c : holder.connections) ::PrintLn(c.server); ::PrintLn("");
106       holder.connections.TakeOut(this);
107       /*
108       PrintLn(server, " Connection Closed (", holder.connections.count, ") opened");
109       printf("Now we have %d connections:\n", holder.connections.count);
110       for(c : holder.connections)
111       {
112          ::PrintLn(c.server);
113       }
114       ::PrintLn("");
115       */
116       delete server;
117    }
118
119    void OnDisconnect(int code)
120    {
121       connectionsMutex.Wait();
122       if(file)
123          delete file.connection; // This decrements the file's reference
124
125       connectionsMutex.Release();
126
127       // PrintLn(server, " Disconnected Us");
128
129       delete this;         // The 'connections' reference
130    }
131
132    uint Open_OnReceive(const byte * buffer, uint count)
133    {
134       HTTPFile file = this.file;
135       int pos = 0;
136       int c;
137       while(!file.done)
138       {
139          bool gotEndLine = false;
140          for(c = 0; c<(int)count-1; c++)
141          {
142             if(buffer[c] == '\r' && buffer[c+1] == '\n')
143             {
144                gotEndLine = true;
145                break;
146             }
147          }
148          if(!gotEndLine)
149             // Incomplete packet
150             return pos;
151          if(c<count)
152          {
153             const char * string = (const char *)buffer;
154
155 #ifdef _DEBUG
156             fwrite(buffer, 1, c, stdout);
157             puts("");
158 #endif
159
160             if(!c)
161             {
162                //if(file.openStarted)
163                   file.done = true;
164             }
165             else
166             {
167                //file.openStarted = true;
168                if((string = GetString((const char *)buffer, "HTTP/1.1 ", count)) ||
169                   (string = GetString((const char *)buffer, "HTTP/1.0 ", count)))
170                {
171                   file.status = atoi(string);
172                }
173                else if((string = GetString((const char *)buffer, "Transfer-Encoding: ", count)))
174                {
175                   if(!strnicmp(string, "chunked", strlen("chunked")))
176                   {
177                      file.chunked = true;
178                   }
179                }
180                else if((string = GetString((const char *)buffer, "Content-Length: ", count)))
181                {
182                   file.totalSize = atoi(string);
183                   file.totalSizeSet = true;
184                }
185                else if((string = GetString((const char *)buffer, "Content-Type: ", count)))
186                {
187                   char * cr = strstr(string, "\r");
188                   char * lf = strstr(string, "\n");
189                   int len;
190                   if(cr)
191                      len = cr - string;
192                   else if(lf)
193                      len = lf - string;
194                   else
195                      len = strlen(string);
196
197                   file.contentType = new char[len+1];
198                   memcpy(file.contentType, string, len);
199                   file.contentType[len] = 0;
200                }
201                else if((string = GetString((const char *)buffer, "Content-disposition: ", count)))
202                {
203                   char * cr = strstr(string, "\r");
204                   char * lf = strstr(string, "\n");
205                   int len;
206                   if(cr)
207                      len = cr - string;
208                   else if(lf)
209                      len = lf - string;
210                   else
211                      len = strlen(string);
212
213                   file.contentDisposition = new char[len+1];
214                   memcpy(file.contentDisposition, string, len);
215                   file.contentDisposition[len] = 0;
216                }
217                else if((string = GetString((const char *)buffer, "Connection: ", count)))
218                {
219                   if(!strnicmp(string, "close", strlen("close")))
220                   {
221                      file.close = true;
222                   }
223                }
224                else if((string = GetString((const char *)buffer, "Location: ", count)))
225                {
226                   if(file.relocation)
227                   {
228                      strncpy(file.relocation, (const char *)buffer + 10, c - 10);
229                      file.relocation[c - 10] = '\0';
230                   }
231                }
232             }
233             // return c+2;
234             if(buffer[c] == '\r') c++;
235             if(buffer[c] == '\n') c++;
236             pos += c;
237             count -= c;
238             buffer += c;
239
240          }
241          else
242             break;
243       }
244
245       return pos;
246    }
247
248    uint Read_OnReceive(const byte * buffer, uint count)
249    {
250       HTTPFile file = this.file;
251       if(file)
252       {
253          int read;
254          if(file.chunked && !file.chunkSize)
255          {
256             int pos = 0, c = 0;
257             const char * string = null;
258             bool ready = false;
259             while(pos + c < (int)count-3)
260             {
261                if(buffer[pos+c] == '\r' && buffer[pos+c+1] == '\n')
262                {
263                   if(string)
264                   {
265                      ready = true;
266                      pos += c + 2;
267                      if(buffer[pos] == '\r' && buffer[pos+1] =='\n') pos+= 2;
268                      break;
269                   }
270                   else
271                   {
272                      pos += 2;
273                      c = 0;
274                   }
275                }
276                else
277                {
278                   if(!string)
279                      string = (const char *)buffer + pos;
280                   c++;
281                }
282             }
283             if(ready)
284             {
285                file.chunkSize = strtol(string, null, 16);
286                if(!file.chunkSize)
287                {
288                   file.connection.file = null;
289                   if(file.close)
290                      file.connection.Disconnect(0);
291                   delete file.connection; // This decrements the file's reference
292                }
293             }
294             return pos;
295          }
296
297          read = Min(count, HTTPFILE_BUFFERSIZE - file.bufferCount);
298          if(file.chunked)
299          {
300             read = Min(read, file.chunkSize);
301             file.chunkSize -= read;
302          }
303          if(read)
304          {
305             memcpy(file.buffer + file.bufferCount, buffer, read);
306             file.bufferCount += read;
307          }
308          return read;
309       }
310       else
311          return count;
312    }
313 }
314
315 public class HTTPFile : File
316 {
317    bool reuseConnection;
318    reuseConnection = true;
319 public:
320    property bool reuseConnection
321    {
322       set { reuseConnection = value; }
323       get { return reuseConnection; }
324    }
325    property String contentType
326    {
327       get { return contentType; }
328    }
329    property String contentDisposition
330    {
331       get { return contentDisposition; }
332    }
333
334    bool OpenURL(const char * name, const char * referer, char * relocation)
335    {
336       return RetrieveHead(name, referer, relocation, false);
337    }
338
339 private:
340
341    bool RetrieveHead(const char * name, const char * referer, char * relocation, bool askBody)
342    {
343       bool result = false;
344       const String http, https;
345       if(!this || !name) return false;
346       http = strstr(name, "http://");
347       if(http != name) http = null;
348       https = http ? null : strstr(name, "https://"); if(https && https != name) https = null;
349
350       askedBody = askBody;
351
352       done = false;
353       delete contentType;
354       delete contentDisposition;
355       // ::PrintLn("Opening ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID(), "\n");
356       if(this && (http || https))
357       {
358          char server[1024];
359          char msg[1024];
360          int len;
361          const char * serverStart = http ? ecere::net::GetString(name, "http://", 0) : ecere::net::GetString(name, "https://", 0);
362          char * fileName = strstr(serverStart, "/");
363          int port = https ? 443 : 80;
364          char * colon;
365          bool reuse = false;
366          HTTPConnection connection = null;
367          close = false;
368
369          if(fileName)
370          {
371             fileName++;
372             memcpy(server, serverStart, fileName - serverStart - 1);
373             server[fileName - serverStart - 1] = '\0';
374          }
375          else
376             strcpy(server, serverStart);
377
378          if(relocation && !fileName && name[strlen(name)-1] != '/')
379          {
380             strcpy(relocation, http ? "http://" : "https://");
381             strcat(relocation, server);
382             strcat(relocation, "/");
383          }
384
385          colon = strchr(server, ':');
386          if(colon)
387          {
388             port = atoi(colon+1);
389             *colon = '\0';
390          }
391
392          connectionsMutex.Wait();
393          if(this.connection)
394          {
395             this.connection.file = null;
396             if(close)
397                this.connection.Disconnect(0);
398             delete this.connection;
399          }
400
401          if(chunked)
402          {
403             while(this.connection && this.connection.connected && this.connection.file)
404             {
405                connectionsMutex.Release();
406                this.connection.Process();
407                connectionsMutex.Wait();
408             }
409          }
410
411          if(reuseConnection)
412          {
413             bool retry = true;
414             while(retry)
415             {
416                retry = false;
417                connection = null;
418                for(c : holder.connections)
419                {
420                   if(!strcmpi(c.server, server) && c.port == port && c.secure == (https ? true : false))
421                   {
422                      if(!c.file && c.connected)
423                      {
424                         connection = c;      // TOFIX: 'incref c' doesn't work
425                         incref connection;      // HTTPFile reference if we keep it
426                         connectionsMutex.Release();
427                         connection.ProcessTimeOut(0.000001);
428                         connectionsMutex.Wait();
429                         if(!connection.connected || connection.file)
430                         {
431                            // We're disconnected or reused already...
432                            retry = true;
433                            delete connection;
434                         }
435                         break;
436                      }
437                   }
438                }
439             }
440             if(connection)
441             {
442                // ::PrintLn("Reusing Connection ", (uint64)connection, " for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID(), "\n");
443                reuse = true;
444                connection.file = this;
445             }
446          }
447
448       tryagain:
449          if(!connection)
450          {
451             char ipAddress[1024];
452             connection = HTTPConnection
453             {
454 #ifndef ECERE_NOSSL
455                autoEstablish = https ? true : false
456 #endif
457             };
458             incref connection;      // HTTPFile reference on success
459
460             connection.file = this;
461
462             connectionsMutex.Release();
463
464             if(serverNameCache.Resolve(server, ipAddress))
465             {
466                // ::PrintLn("No Connection - Connecting ", (uint64)connection, " for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID());
467                if(connection.Connect(ipAddress /*server*/, port))
468                {
469                   //::PrintLn("Successfully Connected for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID());
470                   //::PrintLn("Waiting on connectionsMutex for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID());
471                   connectionsMutex.Wait();
472
473                   connection.server = CopyString(server);
474                   connection.port = port;
475                   connection.secure = https ? true : false;
476
477                   //::PrintLn("Got connectionsMutex for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID());
478                   holder.connections.Add(connection);
479                   /*
480                   printf("Now we have %d connections:\n", holder.connections.count);
481                   for(c : holder.connections)
482                   {
483                      String s = c.server;
484                      ::Print("Server: ");
485                      ::PrintLn(c.server);
486                   }
487                   ::PrintLn("");
488                   */
489                   incref connection;   // Global List Reference
490                }
491                else
492                {
493                   // ::PrintLn("Connection Failed for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID());
494                   connectionsMutex.Wait();
495                   delete connection;
496                }
497             }
498             else
499             {
500                connectionsMutex.Wait();
501                delete connection;
502             }
503          }
504
505          if(connection)
506          {
507             incref connection;      // local reference
508
509             connection.OnReceive = HTTPConnection::Open_OnReceive;
510             connection.file = this;
511             this.connection = connection;
512             this.relocation = relocation;
513             //openStarted = false;
514
515             totalSizeSet = false;   // HEAD will sometimes give you 0!
516             strcpy(msg, askBody ? "GET /" : "HEAD /");
517
518             if(fileName)
519             {
520                byte ch;
521                int c;
522                len = strlen(msg);
523                for(c = 0; (ch = fileName[c]); c++)
524                {
525                   if(ch <= 32 || ch > 128)
526                   {
527                      byte nibble;
528                      msg[len++] = '%';
529                      nibble = (ch & 0xF0) >> 4;
530                      msg[len++] = (nibble > 9) ? (nibble - 10 + 'a') : (nibble + '0');
531                      nibble = ch & 0x0F;
532                      msg[len++] = (nibble > 9) ? (nibble - 10 + 'a') : (nibble + '0');
533                   }
534                   else
535                      msg[len++] = ch;
536                }
537                msg[len] = '\0';
538             }
539
540             strcat(msg, " HTTP/1.1\r\nHost: ");
541             //strcat(msg, " HTTP/1.0\r\nHost: ");
542             strcat(msg, server);
543             strcat(msg, "\r\n");
544             strcat(msg, "Accept-Charset: UTF-8\r\n");
545             //strcat(msg, "Accept-Charset: ISO-8859-1\r\n");
546             strcat(msg, "Connection: Keep-Alive\r\n");
547             if(referer)
548             {
549                strcat(msg, "Referer: ");
550                strcat(msg, referer);
551                strcat(msg, "\r\n");
552             }
553             strcat(msg, "\r\n");
554             len = strlen(msg);
555
556             //::PrintLn("Releasing connectionsMutex before GET for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID());
557             connectionsMutex.Release();
558
559             // ::PrintLn("Sending GET for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID(), "\n");
560             connection.Send(msg, len);
561
562             while(this.connection && this.connection.connected && !done)
563             {
564                this.connection.Process();
565             }
566             //::PrintLn("Got DONE for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID(), "\n");
567
568             if(this.connection)
569             {
570                if(name != location)
571                {
572                   delete location;
573                   location = CopyString(name);
574                }
575                if(status == 200 || (!status && totalSizeSet))
576                {
577                   if(askBody)
578                      this.connection.OnReceive = HTTPConnection::Read_OnReceive;
579
580                   result = true;
581                   connectionsMutex.Wait();
582                }
583                else
584                {
585                   if(askBody)
586                   {
587                      if(chunked)
588                      {
589                         //bool wait = false;
590                         this.connection.OnReceive = HTTPConnection::Read_OnReceive;
591                         while(!eof)
592                         {
593                            if(!this.connection)
594                               eof = true;
595                            else
596                            {
597                               // First time check if we already have bytes, second time wait for an event
598                               this.connection.Process();
599                               //wait = true;
600                            }
601                         }
602                      }
603                      else if(totalSizeSet)
604                      {
605                         // Is it possible to have Content-Length set but not chunked?
606                         done = false;
607                         this.connection.OnReceive = HTTPConnection::Read_OnReceive;
608                         while(this.connection && this.connection.connected && position + (bufferCount - bufferPos) < totalSize)
609                         {
610                            connection.Process();
611                            position += bufferCount - bufferPos;
612                            bufferCount = 0;
613                            bufferPos = 0;
614                         }
615                      }
616                   }
617
618                   connectionsMutex.Wait();
619                   if(this.connection)
620                   {
621                      this.connection.OnReceive = null;
622                      this.connection.file = null;
623
624                      if(close)
625                      {
626                         this.connection.Disconnect(0);
627                         connection = null;
628                      }
629                   }
630
631                   status = 0;
632                   delete this.connection; // This decrements the file's reference
633                   this.relocation = null;
634                   totalSize = 0;
635                   totalSizeSet = false;
636                   done = false;
637                   eof = false;
638                   position = 0;
639                   bufferPos = 0;
640                   bufferCount = 0;
641                   chunked = false;
642                }
643             }
644             else
645             {
646                connectionsMutex.Wait();
647             }
648             if(reuse && !status && connection && !connection.connected)
649             {
650                delete connection;
651                reuse = false;
652                goto tryagain;
653             }
654             else
655                delete connection;
656          }
657          connectionsMutex.Release();
658       }
659       return result;
660    }
661
662    ~HTTPFile()
663    {
664       delete location;
665       delete contentType;
666       delete contentDisposition;
667       {
668          connectionsMutex.Wait();
669          if(connection)
670          {
671             if(totalSizeSet && askedBody)
672             {
673                done = false;
674                this.connection.OnReceive = HTTPConnection::Read_OnReceive;
675                while(this.connection && this.connection.connected && position + (bufferCount - bufferPos) < totalSize)
676                {
677                   connectionsMutex.Release();
678                   connection.Process();
679                   connectionsMutex.Wait();
680                   position += bufferCount - bufferPos;
681                   bufferCount = 0;
682                   bufferPos = 0;
683                }
684                position = 0;
685             }
686
687             if(connection)
688             {
689                connection.file = null;
690                if(close)
691                   connection.Disconnect(0);
692                delete connection;
693             }
694          }
695          connectionsMutex.Release();
696
697          if(chunked)
698          {
699             while(connection && connection.connected && connection.file)
700             {
701                connectionsMutex.Release();
702                connection.Process();
703                connectionsMutex.Wait();
704             }
705          }
706          //::PrintLn("Done with ", (uint64)this);
707       }
708    }
709
710    int Read(byte * buffer, uint size, uint count)
711    {
712       uint readSize = size * count;
713       uint read = 0;
714       bool wait = false;
715       Time lastTime = GetTime();
716
717       if(!askedBody)
718       {
719          askedBody = true;
720          if(!RetrieveHead(this.location, null, null, true))
721             return 0;
722       }
723
724       if(totalSizeSet && position >= totalSize)
725          eof = true;
726       while(!eof && read < readSize && !aborted)
727       {
728          uint numbytes = bufferCount - bufferPos;
729          numbytes = Min(numbytes, readSize - read);
730          if(totalSizeSet)
731             numbytes = Min(numbytes, totalSize - position);
732
733          if(numbytes)
734          {
735             lastTime = GetTime();
736             memcpy(buffer + read, this.buffer + bufferPos, numbytes);
737             bufferPos += numbytes;
738             position += numbytes;
739             read += numbytes;
740
741             lastTime = GetTime();
742
743             if(bufferPos > HTTPFILE_BUFFERSIZE / 2)
744             {
745                // Shift bytes back to beginning of buffer
746                uint shift = bufferCount - bufferPos;
747                if(shift)
748                   memmove(this.buffer, this.buffer + bufferPos, shift);
749                bufferCount -= bufferPos;
750                bufferPos = 0;
751             }
752          }
753          else
754          {
755             if(!connection || !connection.connected)
756                eof = true;
757             else
758             {
759                // First time check if we already have bytes, second time wait for an event
760                connection.Process();
761                if(wait && bufferCount - bufferPos == 0 && GetTime() - lastTime > 5)
762                   eof = true;
763                wait = true;
764             }
765          }
766          if(totalSizeSet && position >= totalSize)
767             eof = true;
768       }
769       return read / size;
770    }
771
772    int Write(const byte * buffer, uint size, uint count)
773    {
774       return 0;
775    }
776
777    bool Getc(char * ch)
778    {
779       int read = Read(ch, 1, 1);
780       return !eof && read != 0;
781    }
782
783    bool Putc(char ch)
784    {
785       return false;
786    }
787
788    bool Puts(const char * string)
789    {
790       return false;
791    }
792
793    bool Seek(int pos, FileSeekMode mode)
794    {
795       if(mode == start && bufferPos == 0 && pos <= bufferCount && pos >= 0)
796       {
797          bufferPos = pos;
798          return true;
799       }
800       else if(mode == current && bufferPos == 0 && ((int)position + pos) <= bufferCount && ((int)position + pos) >= 0)
801       {
802          bufferPos = position + pos;
803          return true;
804       }
805       else if(mode == end && totalSizeSet && bufferPos == 0 && bufferCount == totalSize && ((int)totalSize - pos) <= bufferCount && ((int)totalSize - pos) >= 0)
806       {
807          bufferPos = totalSize - pos;
808          return true;
809       }
810       return false;
811    }
812
813    uint Tell()
814    {
815       return position;
816    }
817
818    bool Eof()
819    {
820       return eof;
821    }
822
823    uint GetSize()
824    {
825       return totalSize;
826    }
827
828    void Abort()
829    {
830       aborted = true;
831    }
832 private:
833
834    bool askedBody;
835
836    HTTPConnection connection;
837    uint position;
838    bool done;
839    bool eof;
840    int status;
841    uint totalSize;
842    bool chunked;
843    bool close;
844    uint chunkSize;
845    char * relocation;
846    String location;
847
848    // Buffering...
849    byte buffer[HTTPFILE_BUFFERSIZE];
850    uint bufferPos;
851    uint bufferCount;
852    bool aborted;
853    bool totalSizeSet;
854
855    String contentType;
856    String contentDisposition;
857 }
858
859 public HTTPFile FileOpenURL(const char * name)
860 {
861    HTTPFile f { };
862    if(f.OpenURL(name, null, null))
863       return f;
864    else
865    {
866       delete f;
867       return null;
868    }
869 }
870
871 #endif