wip II
[sdk] / ecere / src / net / HTTPFile.ec
1 #ifndef ECERE_NONET
2
3 #include <stdio.h>
4
5 import "List"
6 import "network"
7 #ifndef ECERE_NOSSL
8 import "SSLSocket"
9 #endif
10
11 class ConnectionsHolder
12 {
13    List<HTTPConnection> connections { };
14
15    ~ConnectionsHolder()
16    {
17       HTTPConnection c;
18       while((c = connections[0]))
19          delete c;   // The HTTPConnection destructor will take out from the list
20    }
21 }
22
23 static ConnectionsHolder holder { };
24
25 namespace net;
26
27 /*static */define HTTPFILE_BUFFERSIZE = 65536;
28
29 static Mutex connectionsMutex { };
30
31 static class ServerNode : BTNode
32 {
33    char address[24];
34    bool resolved;
35    ~ServerNode()
36    {
37       char * name = (char *)key;
38       delete name;
39    }
40 }
41
42 static class ServerNameCache
43 {
44    BinaryTree servers
45    {
46       CompareKey = (void *)BinaryTree::CompareString;
47    };
48    Mutex mutex { };
49
50    ~ServerNameCache()
51    {
52       ServerNode server;
53       while((server = (ServerNode)servers.root))
54       {
55          servers.Remove(server);
56          delete server;
57       }
58    }
59    bool Resolve(char * host, char * address)
60    {
61       ServerNode server;
62       mutex.Wait();
63       server = (ServerNode)servers.FindString(host);
64       if(!server)
65       {
66          server = ServerNode { key = (uintptr)CopyString(host) };
67          servers.Add(server);
68          server.resolved = GetAddressFromName(host, server.address);
69       }
70       mutex.Release();
71       if(server.resolved)
72          strcpy(address, server.address);
73       return server.resolved;
74    }
75 };
76
77 static ServerNameCache serverNameCache { };
78
79 static char * GetString(char * string, char * what, int count)
80 {
81    int c;
82    for(c = 0; what[c]; c++)
83    {
84       if((count && c >= count) || (string[c] != what[c] && tolower(string[c]) != tolower(what[c])))
85          return null;
86    }
87    return string + c;
88 }
89
90 #ifdef ECERE_NOSSL
91 private class HTTPConnection : Socket
92 #else
93 private class HTTPConnection : SSLSocket
94 #endif
95 {
96    class_no_expansion;
97    char * server;
98    int port;
99    HTTPFile file;
100    bool secure;
101
102    processAlone = true;
103
104    ~HTTPConnection()
105    {
106
107       // printf("Before TakeOut we have %d connections:\n", holder.connections.count); for(c : holder.connections) ::PrintLn(c.server); ::PrintLn("");
108       holder.connections.TakeOut(this);
109       /*
110       PrintLn(server, " Connection Closed (", holder.connections.count, ") opened");
111       printf("Now we have %d connections:\n", holder.connections.count);
112       for(c : holder.connections)
113       {
114          ::PrintLn(c.server);
115       }
116       ::PrintLn("");
117
118
119
120       */
121       delete server;
122    }
123
124    void OnDisconnect(int code)
125    {
126       connectionsMutex.Wait();
127       if(file)
128          delete file.connection; // This decrements the file's reference
129
130       connectionsMutex.Release();
131
132       // PrintLn(server, " Disconnected Us");
133
134       delete this;         // The 'connections' reference
135    }
136
137    uint Open_OnReceive(const byte * buffer, uint count)
138    {
139       HTTPFile file = this.file;
140       int pos = 0;
141       int c;
142       while(!file.done)
143       {
144          bool gotEndLine = false;
145          for(c = 0; c<(int)count-1; c++)
146          {
147             if(buffer[c] == '\r' && buffer[c+1] == '\n')
148             {
149                gotEndLine = true;
150                break;
151             }
152          }
153          if(!gotEndLine)
154             // Incomplete packet
155             return pos;
156          if(c<count)
157          {
158             char * string = (char *)buffer;
159
160             /*
161             char line[1024];
162             memcpy(line, buffer, c+2);
163             line[c+2] = 0;
164             Log(line);
165             */
166             
167 #ifdef _DEBUG
168             fwrite(buffer, 1, c, stdout);
169             puts("");
170 #endif
171
172             if(!c)
173             {
174                //if(file.openStarted)
175                   file.done = true;
176             }
177             else
178             {
179                //file.openStarted = true;
180                if((string = GetString((char *)buffer, "HTTP/1.1 ", count)) ||
181                   (string = GetString((char *)buffer, "HTTP/1.0 ", count)))
182                {
183                   file.status = atoi(string);
184                }
185                else if(string = GetString((char *)buffer, "Transfer-Encoding: ", count))
186                {
187                   if(!strnicmp(string, "chunked", strlen("chunked")))
188                   {
189                      file.chunked = true;
190                   }
191                }
192                else if(string = GetString((char *)buffer, "Content-Length: ", count))
193                {
194                   file.totalSize = atoi(string);
195                   file.totalSizeSet = true;
196                }
197                else if(string = GetString((char *)buffer, "Content-Type: ", count))
198                {
199                   char * cr = strstr(string, "\r");
200                   char * lf = strstr(string, "\n");
201                   int len;
202                   if(cr)
203                      len = cr - string;
204                   else if(lf)
205                      len = lf - string;
206                   else
207                      len = strlen(string);
208
209                   file.contentType = new char[len+1];
210                   memcpy(file.contentType, string, len);
211                   file.contentType[len] = 0;
212                }
213                else if(string = GetString((char *)buffer, "Content-disposition: ", count))
214                {
215                   char * cr = strstr(string, "\r");
216                   char * lf = strstr(string, "\n");
217                   int len;
218                   if(cr)
219                      len = cr - string;
220                   else if(lf)
221                      len = lf - string;
222                   else
223                      len = strlen(string);
224
225                   file.contentDisposition = new char[len+1];
226                   memcpy(file.contentDisposition, string, len);
227                   file.contentDisposition[len] = 0;
228                }
229                else if(string = GetString((char *)buffer, "Connection: ", count))
230                {
231                   if(!strnicmp(string, "close", strlen("close")))
232                   {
233                      file.close = true;
234                   }
235                }
236                else if(string = GetString((char *)buffer, "Location: ", count))
237                {
238                   if(file.relocation)
239                   {
240                      strncpy(file.relocation, buffer + 10, c - 10);
241                      file.relocation[c - 10] = '\0';
242                   }
243                }
244             }
245             // return c+2;
246             if(buffer[c] == '\r') c++;
247             if(buffer[c] == '\n') c++;
248             pos += c;
249             count -= c;
250             buffer += c;
251             
252          }
253          else
254             break;
255       }
256       
257       return pos;
258    }
259
260    uint Read_OnReceive(const byte * buffer, uint count)
261    {
262       HTTPFile file = this.file;
263       if(file)
264       {
265          int read;
266          if(file.chunked && !file.chunkSize)
267          {
268             int pos = 0, c = 0;
269             char * string = null;
270             bool ready = false;
271             while(pos + c < (int)count-3)
272             {
273                if(buffer[pos+c] == '\r' && buffer[pos+c+1] == '\n')
274                {
275                   if(string)
276                   {
277                      ready = true;
278                      pos += c + 2;
279                      if(buffer[pos] == '\r' && buffer[pos+1] =='\n') pos+= 2;
280                      break;
281                   }
282                   else
283                   {
284                      pos += 2;
285                      c = 0;
286                   }
287                }
288                else
289                {
290                   if(!string)
291                      string = buffer + pos;
292                   c++;
293                }
294             }
295             if(ready)
296             {
297                file.chunkSize = strtol(string, null, 16);
298                if(!file.chunkSize)
299                {
300                   //connection.file = null;
301                   file.connection.file = null;
302                   delete file.connection; // This decrements the file's reference
303                }
304             }
305             return pos;
306          }
307
308          read = Min(count, HTTPFILE_BUFFERSIZE - file.bufferCount);
309          if(file.chunked)
310          {
311             read = Min(read, file.chunkSize);
312             file.chunkSize -= read;
313          }
314          if(read)
315          {
316             memcpy(file.buffer + file.bufferCount, buffer, read);
317             // fwrite(file.buffer, 1, read, stdout);
318             file.bufferCount += read;
319             /*
320             if(GetCurrentThreadID() == network.mainThreadID)
321                fwrite(buffer, 1, read, stdout);
322             */
323          }
324          return read;
325       }
326       else
327          return count;
328    }
329 }
330
331 public class HTTPFile : File
332 {
333    bool reuseConnection;
334    reuseConnection = true;
335 public:
336    property bool reuseConnection
337    {
338       set { reuseConnection = value; }
339       get { return reuseConnection; }
340    }
341    property String contentType
342    {
343       get { return contentType; }
344    }
345    property String contentDisposition
346    {
347       get { return contentDisposition; }
348    }
349
350    bool OpenURL(char * name, char * referer, char * relocation)
351    {
352       return RetrieveHead(name, referer, relocation, false);
353    }
354
355 private:
356
357    bool RetrieveHead(char * name, char * referer, char * relocation, bool askBody)
358    {
359       bool result = false;
360       String http, https;
361       if(!this || !name) return false;
362       http = strstr(name, "http://");
363       if(http != name) http = null;
364       https = http ? null : strstr(name, "https://"); if(https && https != name) https = null;
365
366       askedBody = askBody;
367
368       done = false;
369       delete contentType;
370       delete contentDisposition;
371       // ::PrintLn("Opening ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID(), "\n");
372       if(this && (http || https))
373       {
374          char server[1024];
375          char msg[1024];
376          int len;
377          char * serverStart = http ? ecere::net::GetString(name, "http://", 0) : ecere::net::GetString(name, "https://", 0);
378          char * fileName = strstr(serverStart, "/");
379          int port = https ? 443 : 80;
380          char * colon;
381          bool reuse = false;
382          HTTPConnection connection = null;
383          close = false;
384
385          if(fileName) 
386          {
387             fileName++;
388             memcpy(server, serverStart, fileName - serverStart - 1);
389             server[fileName - serverStart - 1] = '\0';
390          }
391          else
392             strcpy(server, serverStart);
393
394          if(relocation && !fileName && name[strlen(name)-1] != '/')
395          {
396             strcpy(relocation, http ? "http://" : "https://");
397             strcat(relocation, server);
398             strcat(relocation, "/");
399          }
400
401          colon = strchr(server, ':');
402          if(colon)
403          {
404             port = atoi(colon+1);
405             *colon = '\0';
406          }
407
408          connectionsMutex.Wait();
409          if(this.connection)
410          {
411             this.connection.file = null;
412             if(close)
413                this.connection.Disconnect(0);
414             delete this.connection;
415          }
416
417          if(chunked)
418          {
419             while(this.connection && this.connection.connected && this.connection.file)
420             {
421                connectionsMutex.Release();
422                this.connection.Process();
423                connectionsMutex.Wait();
424             }
425          }
426
427          if(reuseConnection)
428          {
429             bool retry = true;
430             while(retry)
431             {
432                retry = false;
433                connection = null;
434                for(c : holder.connections)
435                {
436                   if(!strcmpi(c.server, server) && c.port == port && c.secure == (https ? true : false))
437                   {
438                      if(!c.file && c.connected)
439                      {
440                         connection = c;      // TOFIX: 'incref c' doesn't work
441                         incref connection;      // HTTPFile reference if we keep it
442                         connectionsMutex.Release();
443                         connection.ProcessTimeOut(0.000001);
444                         connectionsMutex.Wait();
445                         if(!connection.connected || connection.file)
446                         {
447                            // We're disconnected or reused already...
448                            retry = true;
449                            delete connection;
450                         }
451                         break;
452                      }
453                   }
454                }
455             }
456             if(connection)
457             {
458                // ::PrintLn("Reusing Connection ", (uint64)connection, " for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID(), "\n");
459                reuse = true;
460                connection.file = this;
461             }
462          }
463
464       tryagain:
465          if(!connection)
466          {
467             char ipAddress[1024];
468             connection = HTTPConnection
469             {
470 #ifndef ECERE_NOSSL
471                autoEstablish = https ? true : false
472 #endif
473             };
474             incref connection;      // HTTPFile reference on success
475             
476             connection.file = this;
477
478             connectionsMutex.Release();
479
480             if(serverNameCache.Resolve(server, ipAddress))
481             {
482                // ::PrintLn("No Connection - Connecting ", (uint64)connection, " for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID());
483                if(connection.Connect(ipAddress /*server*/, port))
484                {
485                   //::PrintLn("Successfully Connected for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID());
486                   //::PrintLn("Waiting on connectionsMutex for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID());
487                   connectionsMutex.Wait();
488
489                   connection.server = CopyString(server);
490                   connection.port = port;
491                   connection.secure = https ? true : false;
492
493                   //::PrintLn("Got connectionsMutex for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID());
494                   holder.connections.Add(connection);
495                   /*
496                   printf("Now we have %d connections:\n", holder.connections.count);
497                   for(c : holder.connections)
498                   {
499                      String s = c.server;
500                      ::Print("Server: ");
501                      ::PrintLn(c.server);
502                   }
503                   ::PrintLn("");
504                   */
505                   incref connection;   // Global List Reference
506                }
507                else
508                {
509                   // ::PrintLn("Connection Failed for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID());
510                   connectionsMutex.Wait();
511                   delete connection;
512                }
513             }
514             else
515             {
516                connectionsMutex.Wait();
517                delete connection;
518             }
519          }
520
521          if(connection)
522          {
523             incref connection;      // local reference
524
525             connection.OnReceive = HTTPConnection::Open_OnReceive;
526             connection.file = this;
527             this.connection = connection;
528             this.relocation = relocation;
529             //openStarted = false;
530                   
531             totalSizeSet = false;   // HEAD will sometimes give you 0!
532             strcpy(msg, askBody ? "GET /" : "HEAD /");
533
534             if(fileName)
535             {
536                byte ch;
537                int c;
538                len = strlen(msg);
539                for(c = 0; (ch = fileName[c]); c++)
540                {
541                   if(ch <= 32 || ch > 128)
542                   {
543                      byte nibble;
544                      msg[len++] = '%';
545                      nibble = (ch & 0xF0) >> 4;
546                      msg[len++] = (nibble > 9) ? (nibble - 10 + 'a') : (nibble + '0');
547                      nibble = ch & 0x0F;
548                      msg[len++] = (nibble > 9) ? (nibble - 10 + 'a') : (nibble + '0');
549                   }
550                   else
551                      msg[len++] = ch;
552                }
553                msg[len] = '\0';
554             }
555
556             strcat(msg, " HTTP/1.1\r\nHost: ");
557             //strcat(msg, " HTTP/1.0\r\nHost: ");
558             strcat(msg, server);
559             strcat(msg, "\r\n");
560             strcat(msg, "Accept-Charset: UTF-8\r\n");
561             //strcat(msg, "Accept-Charset: ISO-8859-1\r\n");
562             strcat(msg, "Connection: Keep-Alive\r\n");
563             if(referer)
564             {
565                strcat(msg, "Referer: ");
566                strcat(msg, referer);
567                strcat(msg, "\r\n");
568             }
569             /*else
570             {
571                strcat(msg, "Referer: ");
572                strcat(msg, server);
573                strcat(msg, "\r\n");
574             }*/
575             strcat(msg, "\r\n");
576             len = strlen(msg);
577             
578             //::PrintLn("Releasing connectionsMutex before GET for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID());
579             connectionsMutex.Release();
580
581             // ::PrintLn("Sending GET for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID(), "\n");
582             connection.Send(msg, len);
583
584             while(this.connection && this.connection.connected && !done)
585             {
586                this.connection.Process();
587             }
588             //::PrintLn("Got DONE for ", name, " ", (uint64)this, " in thread ", GetCurrentThreadID(), "\n");
589
590             if(this.connection)
591             {
592                if(name != location)
593                {
594                   delete location;
595                   location = CopyString(name);
596                }
597                if(status == 200 || (!status && totalSizeSet))
598                {
599                   if(askBody)
600                      this.connection.OnReceive = HTTPConnection::Read_OnReceive;
601
602                   result = true;
603                   connectionsMutex.Wait();
604                }
605                else
606                {
607                   if(askBody)
608                   {
609                      if(chunked)
610                      {
611                         bool wait = false;
612                         this.connection.OnReceive = HTTPConnection::Read_OnReceive;
613                         while(!eof)
614                         {
615                            if(!this.connection)
616                               eof = true;
617                            else
618                            {
619                               // First time check if we already have bytes, second time wait for an event
620                               this.connection.Process();
621                               wait = true;
622                            }
623                         }
624                      }
625                      else if(totalSizeSet)
626                      {
627                         done = false;
628                         this.connection.OnReceive = HTTPConnection::Read_OnReceive;
629                         while(this.connection && this.connection.connected && position < totalSize)
630                         {
631                            connection.Process();
632                            position += bufferCount;
633                            bufferCount = 0;
634                         }
635                      }
636                   }
637                   
638                   connectionsMutex.Wait();
639                   if(this.connection)
640                   {
641                      this.connection.OnReceive = null;
642                      this.connection.file = null;
643                      
644                      if(close)
645                      {
646                         this.connection.Disconnect(0);
647                         connection = null;
648                      }
649                   }
650                   
651                   status = 0;
652                   delete this.connection; // This decrements the file's reference
653                   this.relocation = null;
654                   totalSize = 0;
655                   totalSizeSet = false;
656                   done = false;
657                   eof = false;
658                   position = 0;
659                   bufferPos = 0;
660                   bufferCount = 0;
661                   chunked = false;
662                }
663             }
664             else
665             {
666                connectionsMutex.Wait();
667             }
668             if(reuse && !status && connection && !connection.connected)
669             {
670                delete connection;
671                reuse = false;
672                goto tryagain;
673             }
674             else
675                delete connection;
676          }
677          connectionsMutex.Release();
678       }
679       return result;
680    }
681
682    ~HTTPFile()
683    {
684       delete location;
685       delete contentType;
686       delete contentDisposition;
687       {
688          connectionsMutex.Wait();
689          if(connection)
690          {
691             if(totalSizeSet && askedBody)
692             {
693                done = false;
694                this.connection.OnReceive = HTTPConnection::Read_OnReceive;
695                while(this.connection && this.connection.connected && position + (bufferCount - bufferPos) < totalSize)
696                {
697                   connectionsMutex.Release();
698                   connection.Process();
699                   connectionsMutex.Wait();
700                   position += bufferCount;
701                   bufferCount = 0;
702                }
703                position = 0;
704             }
705
706             if(connection)
707             {
708                connection.file = null;
709                if(close)
710                   connection.Disconnect(0);
711                delete connection;
712             }
713          }
714          connectionsMutex.Release();
715
716          if(chunked)
717          {
718             while(connection && connection.connected && connection.file)
719             {
720                connectionsMutex.Release();
721                connection.Process();
722                connectionsMutex.Wait();
723             }
724          }
725          //::PrintLn("Done with ", (uint64)this);
726       }
727    }
728
729    int Read(byte * buffer, uint size, uint count)
730    {
731       uint readSize = size * count;
732       uint read = 0;
733       bool wait = false;
734       Time lastTime = GetTime();
735
736       if(!askedBody)
737       {
738          askedBody = true;
739          if(!RetrieveHead(this.location, null, null, true))
740             return 0;
741       }
742
743       if(totalSizeSet && position >= totalSize)
744          eof = true;
745       while(!eof && read < readSize && !aborted)
746       {
747          uint numbytes = bufferCount - bufferPos;
748          numbytes = Min(numbytes, readSize - read);
749          if(totalSizeSet)
750             numbytes = Min(numbytes, totalSize - position);
751          
752          if(numbytes)
753          {
754             lastTime = GetTime();
755             memcpy(buffer + read, this.buffer + bufferPos, numbytes);
756             bufferPos += numbytes;
757             position += numbytes;
758             read += numbytes;
759
760             lastTime = GetTime();
761
762             if(bufferPos > HTTPFILE_BUFFERSIZE / 2)
763             {
764                // Shift bytes back to beginning of buffer
765                uint shift = bufferCount - bufferPos;
766                if(shift)
767                   memcpy(this.buffer, this.buffer + bufferPos, shift);
768                bufferCount -= bufferPos;
769                bufferPos = 0;
770             }
771          }
772          else
773          {
774             if(!connection || !connection.connected)
775                eof = true;
776             else
777             {
778                // First time check if we already have bytes, second time wait for an event
779                connection.Process();
780                if(wait && bufferCount - bufferPos == 0 && GetTime() - lastTime > 5)
781                   eof = true;
782                wait = true;
783             }
784          }
785          if(totalSizeSet && position >= totalSize)
786             eof = true;
787       }
788       return read / size;
789    }
790
791    int Write(byte * buffer, uint size, uint count)
792    {
793       return 0;
794    }
795
796    bool Getc(char * ch)
797    {
798       int read = Read(ch, 1, 1);
799       return !eof && read != 0;
800    }
801
802    bool Putc(char ch)
803    {
804       return false;
805    }
806
807    bool Puts(char * string)
808    {
809       return false;
810    }
811
812    bool Seek(int pos, FileSeekMode mode)
813    {
814       if(mode == start && bufferPos == 0 && pos <= bufferCount & pos >= 0)
815       {
816          bufferPos = pos;
817          return true;
818       }
819       else if(mode == current && bufferPos == 0 && (position + pos) <= bufferCount & (position + pos) >= 0)
820       {
821          bufferPos = position + pos;
822          return true;
823       }
824       else if(mode == end && totalSizeSet && bufferPos == 0 && bufferCount == totalSize && (totalSize - pos) <= bufferCount & (totalSize - pos) >= 0)
825       {
826          bufferPos = totalSize - pos;
827          return true;
828       }
829       return false;
830    }
831
832    uint Tell()
833    {
834       return position;
835    }
836
837    bool Eof()
838    {
839       return eof;
840    }
841
842    bool GetSize()
843    {
844       return totalSize;
845    }
846
847    void Abort()
848    {
849       aborted = true;
850    }
851 private:
852
853    bool askedBody;
854
855    HTTPConnection connection;
856    uint position;
857    bool done;
858    bool eof;
859    int status;
860    uint totalSize;
861    bool chunked;
862    bool close;
863    uint chunkSize;
864    char * relocation;
865    String location;
866
867    // Buffering...
868    byte buffer[HTTPFILE_BUFFERSIZE];
869    uint bufferPos;
870    uint bufferCount;
871    bool aborted;
872    bool totalSizeSet;
873
874    String contentType;
875    String contentDisposition;
876 }
877
878 public HTTPFile FileOpenURL(char * name)
879 {
880    HTTPFile f { };
881    if(f.OpenURL(name, null, null))
882       return f;
883    else
884    {
885       delete f;
886       return null;
887    }
888 }
889
890 #endif