WvStreams
wvhttppool.h
1 /* -*- Mode: C++ -*-
2  * Worldvisions Weaver Software:
3  * Copyright (C) 1997-2002 Net Integration Technologies, Inc.
4  *
5  * A fast, easy-to-use, parallelizing, pipelining HTTP/1.1 file retriever.
6  *
7  * Just create a WvHttpPool object, add it to your list, and use pool.addurl()
8  * to get a WvStream* that gives you the file you requested.
9  */
10 #ifndef __WVHTTPPOOL_H
11 #define __WVHTTPPOOL_H
12 
13 #include "ftpparse.h"
14 #include "wvurl.h"
15 #include "wvistreamlist.h"
16 #include "wvstreamclone.h"
17 #include "wvlog.h"
18 #include "wvhashtable.h"
19 #include "wvbufstream.h"
20 #include "wvbuf.h"
21 #include "wvcont.h"
22 #include "wvtcp.h"
23 
24 class WvBufUrlStream;
25 class WvUrlStream;
26 class WvHttpStream;
27 
28 static const WvString DEFAULT_ANON_PW("weasels@");
29 
31 {
32  WvString name, value;
33 
34  WvHTTPHeader(WvStringParm _name, WvStringParm _value)
35  : name(_name), value(_value)
36  {}
37 };
38 
39 
40 DeclareWvDict(WvHTTPHeader, WvString, name);
41 
42 
44 {
45 public:
46  WvUrl url;
47  WvString headers;
48  WvUrlStream *instream;
49  WvBufUrlStream *outstream;
50  WvStream *putstream;
51 
52  bool pipeline_test;
53  bool inuse;
54  bool is_dir;
55  bool create_dirs;
56  WvString method;
57 
58  WvUrlRequest(WvStringParm _url, WvStringParm _method, WvStringParm _headers,
59  WvStream *content_source, bool _create_dirs, bool _pipeline_test);
60  ~WvUrlRequest();
61 
62  void done();
63 };
64 
65 DeclareWvList(WvUrlRequest);
66 
67 
68 struct WvUrlLink
69 {
70  WvString linkname;
71  WvUrl url;
72 
73  WvUrlLink(WvStringParm _linkname, WvStringParm _url)
74  : linkname(_linkname), url(_url)
75  {}
76 };
77 DeclareWvList(WvUrlLink);
78 
79 
81 {
82 public:
83  WvString url;
84  WvString proto;
85  WvUrlLinkList links; // HTML links or FTP directory listing
86 
87  // HTTP stuff...
88  WvString version;
89  int status;
90  WvHTTPHeaderDict headers;
91 
92  WvBufUrlStream() : status(0), headers(10)
93  {}
94  virtual ~WvBufUrlStream()
95  {}
96 
97 public:
98  const char *wstype() const { return "WvBufUrlStream"; }
99 };
100 
101 DeclareWvTable(WvIPPortAddr);
102 
103 
105 {
106 public:
107  class Target
108  {
109  public:
110  WvIPPortAddr remaddr;
111  WvString username;
112 
113  Target(const WvIPPortAddr &_remaddr, WvStringParm _username)
114  : remaddr(_remaddr), username(_username) {}
115 
116  ~Target() {}
117 
118  bool operator== (const Target &n2) const
119  { return (username == n2.username && remaddr == n2.remaddr); }
120  };
121  Target target;
122  static int max_requests;
123 
124 protected:
125  WvLog log;
126  WvUrlRequestList urls, waiting_urls;
127  int request_count;
128  WvUrlRequest *curl; // current url
129  virtual void doneurl() = 0;
130  virtual void request_next() = 0;
131 
132 public:
133  WvUrlStream(const WvIPPortAddr &_remaddr, WvStringParm _username,
134  WvStringParm logname)
135  : WvStreamClone(new WvTCPConn(_remaddr)), target(_remaddr, _username),
136  log(logname, WvLog::Debug)
137  {
138  request_count = 0;
139  curl = NULL;
140  }
141 
142  virtual ~WvUrlStream() {};
143 
144  virtual void close() = 0;
145  void addurl(WvUrlRequest *url);
146  void delurl(WvUrlRequest *url);
147  // only implemented in WvHttpStream
148  virtual size_t remaining()
149  { return 0; }
150 
151  virtual void execute() = 0;
152 
153 public:
154  const char *wstype() const { return "WvUrlStream"; }
155 };
156 
157 unsigned WvHash(const WvUrlStream::Target &n);
158 
159 DeclareWvDict(WvUrlStream, WvUrlStream::Target, target);
160 
161 
162 class WvHttpStream : public WvUrlStream
163 {
164 public:
165  static bool global_enable_pipelining;
166  bool enable_pipelining;
167 
168 private:
169  int pipeline_test_count;
170  bool ssl;
171  bool sent_url_request; // Have we sent a request to the server yet?
172  WvIPPortAddrTable &pipeline_incompatible;
173  WvString http_response, pipeline_test_response;
174  WvDynBuf putstream_data;
175 
176  enum { Unknown, Chunked, ContentLength, Infinity,
177  PostHeadInfinity, PostHeadChunked, PostHeadStream,
178  ChuckInfinity, ChuckChunked, ChuckStream } encoding;
179  size_t bytes_remaining;
180  bool in_chunk_trailer, last_was_pipeline_test, in_doneurl;
181 
182  virtual void doneurl();
183  virtual void request_next();
184  void start_pipeline_test(WvUrl *url);
185  WvString request_str(WvUrlRequest *url, bool keep_alive);
186  void send_request(WvUrlRequest *url);
187  void pipelining_is_broken(int why);
188 
189 public:
190  WvHttpStream(const WvIPPortAddr &_remaddr, WvStringParm _username,
191  bool ssl, WvIPPortAddrTable &_pipeline_incompatible);
192  virtual ~WvHttpStream();
193 
194  virtual void close();
195  virtual void pre_select(SelectInfo &si);
196  virtual bool post_select(SelectInfo &si);
197  virtual void execute();
198  virtual size_t remaining()
199  { return bytes_remaining; }
200 
201 public:
202  const char *wstype() const { return "WvHttpStream"; }
203 };
204 
205 
206 class WvFtpStream : public WvUrlStream
207 {
208  bool logged_in, pasv_acked;
209  WvString password;
210  WvTCPConn *data;
211  time_t last_request_time;
212  bool sure;
213 
214  virtual void doneurl();
215  virtual void request_next();
216 
217  // Disregard all lines that are of the form "xxx-", meaning that another
218  // line follows. Only the last line is important for us.
219  char *get_important_line();
220 
221  // Parse response to "PASV" command and returns a pointer to the address
222  // of the data port (or NULL if it can't parse the response)..
223  // This mucks about with line.
224  WvIPPortAddr *parse_pasv_response(char *line);
225 
226  WvString parse_for_links(char *line);
227 
228  WvCont cont;
229  void* real_execute(void*);
230 
231 public:
232  WvFtpStream(const WvIPPortAddr &_remaddr, WvStringParm _username,
233  WvStringParm _password);
234 
235  virtual void pre_select(SelectInfo &si);
236  virtual bool post_select(SelectInfo &si);
237  virtual void close();
238  virtual void execute();
239 
240 public:
241  const char *wstype() const { return "WvFtpStream"; }
242 };
243 
244 
245 // FIXME: Rename this to WvUrlPool someday.
246 class WvHttpPool : public WvIStreamList
247 {
248  WvLog log;
249  WvResolver dns;
250  WvUrlStreamDict conns;
251  WvUrlRequestList urls;
252  int num_streams_created;
253  bool sure;
254 
255  WvIPPortAddrTable pipeline_incompatible;
256 
257 public:
258  WvHttpPool();
259  virtual ~WvHttpPool();
260 
261  virtual void pre_select(SelectInfo &si);
262  virtual bool post_select(SelectInfo &si);
263  virtual void execute();
264 
265  WvBufUrlStream *addurl(WvStringParm _url, WvStringParm _method = "GET",
266  WvStringParm _headers = "",
267  WvStream *content_source = NULL,
268  bool create_dirs = false);
269 
270  // For URL uploads. create_dirs should be true if you want all
271  // non-existent directories in _url to be created.
272 // WvBufUrlStream *addputurl(WvStringParm _url, WvStringParm _headers,
273 // WvStream *s, bool create_dirs = false);
274 private:
275  void unconnect(WvUrlStream *s);
276 
277 public:
278  bool idle() const
279  { return !urls.count(); }
280 
281 public:
282  const char *wstype() const { return "WvHttpPool"; }
283 };
284 
285 
286 #endif // __WVHTTPPOOL_H
WvFtpStream::close
virtual void close()
Close this stream.
Definition: wvftpstream.cc:93
WvHttpStream
Definition: wvhttppool.h:162
WvFtpStream::execute
virtual void execute()
The callback() function calls execute(), and then calls the user- specified callback if one is define...
Definition: wvftpstream.cc:465
WvHttpStream::post_select
virtual bool post_select(SelectInfo &si)
post_select() is called after ::select(), and returns true if this object is now ready.
Definition: wvhttpstream.cc:319
WvResolver
ASynchronous DNS resolver functions, so that we can do non-blocking lookups.
Definition: wvresolver.h:24
WvUrlStream::Target
Definition: wvhttppool.h:107
WvTCPConn
WvTCPConn tries to make all outgoing connections asynchronously (in the background).
Definition: wvtcp.h:39
WvString
WvString is an implementation of a simple and efficient printable-string class.
Definition: wvstring.h:329
WvFtpStream::pre_select
virtual void pre_select(SelectInfo &si)
pre_select() sets up for eventually calling ::select().
Definition: wvftpstream.cc:133
WvUrlStream::execute
virtual void execute()=0
The callback() function calls execute(), and then calls the user- specified callback if one is define...
WvLog
A WvLog stream accepts log messages from applications and forwards them to all registered WvLogRcv's.
Definition: wvlog.h:56
WvCont
WvCont provides "continuations", which are apparently also known as semi-coroutines.
Definition: wvcont.h:29
WvHttpStream::execute
virtual void execute()
The callback() function calls execute(), and then calls the user- specified callback if one is define...
Definition: wvhttpstream.cc:339
IWvStream::SelectInfo
the data structure used by pre_select()/post_select() and internally by select().
Definition: iwvstream.h:50
WvUrlStream::close
virtual void close()=0
Close this stream.
WvHttpPool::post_select
virtual bool post_select(SelectInfo &si)
post_select() is called after ::select(), and returns true if this object is now ready.
Definition: wvhttppool.cc:150
WvIPPortAddr
An IP+Port address also includes a port number, with the resulting form www.xxx.yyy....
Definition: wvaddr.h:393
WvFtpStream::post_select
virtual bool post_select(SelectInfo &si)
post_select() is called after ::select(), and returns true if this object is now ready.
Definition: wvftpstream.cc:149
WvHttpStream::pre_select
virtual void pre_select(SelectInfo &si)
pre_select() sets up for eventually calling ::select().
Definition: wvhttpstream.cc:301
WvStreamClone
WvStreamClone simply forwards all requests to the "cloned" stream.
Definition: wvstreamclone.h:23
WvBufUrlStream
Definition: wvhttppool.h:80
WvStream
Unified support for streams, that is, sequences of bytes that may or may not be ready for read/write ...
Definition: wvstream.h:24
WvUrlStream
Definition: wvhttppool.h:104
WvFtpStream
Definition: wvhttppool.h:206
WvDynBufBase< unsigned char >
WvUrl
Definition: wvurl.h:16
WvHttpPool::pre_select
virtual void pre_select(SelectInfo &si)
pre_select() sets up for eventually calling ::select().
Definition: wvhttppool.cc:121
WvBufStream
WvBufStream stores data written by write(), and returns it later on in read().
Definition: wvbufstream.h:31
WvHTTPHeader
Definition: wvhttppool.h:30
WvIStreamList
WvStreamList holds a list of WvStream objects – and its select() and callback() functions know how to...
Definition: wvistreamlist.h:20
WvHttpStream::close
virtual void close()
Close this stream.
Definition: wvhttpstream.cc:73
WvHttpPool::execute
virtual void execute()
The callback() function calls execute(), and then calls the user- specified callback if one is define...
Definition: wvhttppool.cc:202
WvUrlRequest
Definition: wvhttppool.h:43
WvHttpPool
Definition: wvhttppool.h:246