WvStreams
wvftpstream.cc
1 /*
2  * Worldvisions Weaver Software:
3  * Copyright (C) 1997-2002 Net Integration Technologies, Inc.
4  *
5  * A fast, easy-to-use, parallelizing, pipelining HTTP/1.1 file retriever.
6  *
7  * See wvhttppool.h.
8  */
9 
10 #ifdef __GNUC__
11 # define alloca __builtin_alloca
12 #else
13 # ifdef _MSC_VER
14 # include <malloc.h>
15 # define alloca _alloca
16 # else
17 # if HAVE_ALLOCA_H
18 # include <alloca.h>
19 # else
20 # ifdef _AIX
21 #pragma alloca
22 # else
23 # ifndef alloca /* predefined by HP cc +Olibcalls */
24 char *alloca ();
25 # endif
26 # endif
27 # endif
28 # endif
29 #endif
30 
31 #include <ctype.h>
32 #include <time.h>
33 #include "wvhttppool.h"
34 #include "wvbufstream.h"
35 #include "wvtcp.h"
36 #include "wvsslstream.h"
37 #include "strutils.h"
38 #include <stdlib.h> // for alloca()... FIXME: which we shouldn't be using!
39 
40 WvFtpStream::WvFtpStream(const WvIPPortAddr &_remaddr, WvStringParm _username,
41  WvStringParm _password)
42  : WvUrlStream(_remaddr, _username, WvString("FTP %s", _remaddr)),
43  cont(wv::bind(&WvFtpStream::real_execute, this, _1))
44 {
45  data = NULL;
46  logged_in = false;
47  password = _password;
48  last_request_time = time(0);
49  alarm(60000); // timeout if no connection, or something goes wrong
50 }
51 
52 
53 void WvFtpStream::doneurl()
54 {
55  log("Done URL: %s\n", curl->url);
56 
57  curl->done();
58  curl = NULL;
59  WVRELEASE(data);
60  urls.unlink_first();
61  last_request_time = time(0);
62  alarm(60000);
63  request_next();
64  // We just processed the last url in the queue,
65  // so go away.
66  if (urls.isempty() && waiting_urls.isempty())
67  close();
68 }
69 
70 
71 void WvFtpStream::request_next()
72 {
73  // don't do a request if we've done too many already or we have none
74  // waiting.
75  if (request_count >= max_requests || waiting_urls.isempty())
76  return;
77 
78  if (!urls.isempty())
79  return;
80 
81  // okay then, we really do want to send a new request.
82  WvUrlRequest *url = waiting_urls.first();
83 
84  waiting_urls.unlink_first();
85 
86  request_count++;
87  log("Request #%s: %s\n", request_count, url->url);
88  urls.append(url, false, "request_url");
89  alarm(0);
90 }
91 
92 
94 {
95  if (isok())
96  log("Closing.\n");
98 
99  if (geterr())
100  {
101  // if there was an error, count the first URL as done. This prevents
102  // retrying indefinitely.
103  if (!curl && !urls.isempty())
104  curl = urls.first();
105  if (!curl && !waiting_urls.isempty())
106  curl = waiting_urls.first();
107  if (curl)
108  log("URL '%s' is FAILED\n", curl->url);
109  if (curl)
110  curl->done();
111  }
112 
113  if (curl)
114  curl->done();
115 }
116 
117 
118 char *WvFtpStream::get_important_line()
119 {
120  char *line;
121  do
122  {
123  line = blocking_getline(-1);
124  if (!line)
125  return NULL;
126  }
127  while (line[3] == '-');
128  log(WvLog::Debug5, ">> %s\n", line);
129  return line;
130 }
131 
132 
134 {
135  SelectRequest oldwant = si.wants;
136 
138 
139  if (data)
140  data->pre_select(si);
141 
142  if (curl && curl->putstream)
143  curl->putstream->pre_select(si);
144 
145  si.wants = oldwant;
146 }
147 
148 
150 {
151  SelectRequest oldwant = si.wants;
152 
153  if (WvUrlStream::post_select(si))
154  return true;
155 
156  if (data && data->post_select(si))
157  return true;
158 
159  if (curl && curl->putstream && curl->putstream->post_select(si))
160  return true;
161 
162  si.wants = oldwant;
163 
164  return false;
165 }
166 
167 
168 void *WvFtpStream::real_execute(void*)
169 {
170  WvString line;
172 
173  if (alarm_was_ticking && ((last_request_time + 60) <= time(0)))
174  {
175  log(WvLog::Debug4, "urls count: %s\n", urls.count());
176  if (urls.isempty())
177  close(); // timed out, but not really an error
178 
179  return 0;
180  }
181 
182  if (!logged_in)
183  {
184  line = get_important_line();
185  if (!line)
186  {
187  seterr("Server not reachable: %s\n",strerror(errno));
188  return 0;
189  }
190 
191  if (strncmp(line, "220", 3))
192  {
193  log("Server rejected connection: %s\n", line);
194  seterr("server rejected connection");
195  return 0;
196  }
197  print("USER %s\r\n", !target.username ? WvString("anonymous") :
198  target.username);
199  line = get_important_line();
200  if (!line)
201  return 0;
202 
203  if (!strncmp(line, "230", 3))
204  {
205  log(WvLog::Info, "Server doesn't need password.\n");
206  logged_in = true; // No password needed;
207  }
208  else if (!strncmp(line, "33", 2))
209  {
210  print("PASS %s\r\n", !password ? DEFAULT_ANON_PW : password);
211 
212  line = get_important_line();
213  if (!line)
214  return 0;
215 
216  if (line[0] == '2')
217  {
218  log(WvLog::Info, "Authenticated.\n");
219  logged_in = true;
220  }
221  else
222  {
223  log("Strange response to PASS command: %s\n", line);
224  seterr("strange response to PASS command");
225  return 0;
226  }
227  }
228  else
229  {
230  log("Strange response to USER command: %s\n", line);
231  seterr("strange response to USER command");
232  return 0;
233  }
234 
235  print("TYPE I\r\n");
236  log(WvLog::Debug5, "<< TYPE I\n");
237  line = get_important_line();
238  if (!line)
239  return 0;
240 
241  if (strncmp(line, "200", 3))
242  {
243  log("Strange response to TYPE I command: %s\n", line);
244  seterr("strange response to TYPE I command");
245  return 0;
246  }
247  }
248 
249  if (!curl && !urls.isempty())
250  {
251  curl = urls.first();
252 
253  print("CWD %s\r\n", curl->url.getfile());
254  line = get_important_line();
255  if (!line)
256  return 0;
257 
258  if (!strncmp(line, "250", 3))
259  {
260  log(WvLog::Debug5, "This is a directory.\n");
261  curl->is_dir = true;
262  }
263 
264  print("PASV\r\n");
265  line = get_important_line();
266  if (!line)
267  return 0;
268  WvIPPortAddr *dataip = parse_pasv_response(line.edit());
269 
270  if (!dataip)
271  return 0;
272 
273  log(WvLog::Debug4, "Data port is %s.\n", *dataip);
274  // Open data connection.
275  data = new WvTCPConn(*dataip);
276  if (!data)
277  {
278  log("Can't open data connection.\n");
279  seterr("can't open data connection");
280  return 0;
281  }
282 
283  if (curl->is_dir)
284  {
285  if (!curl->putstream)
286  {
287  print("LIST %s\r\n", curl->url.getfile());
288  if (curl->outstream)
289  {
290  WvString url_no_pw("ftp://%s%s%s%s", curl->url.getuser(),
291  !!curl->url.getuser() ? "@" : "",
292  curl->url.gethost(),
293  curl->url.getfile());
294  curl->outstream->print("<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML "
295  "4.01//EN\">\n"
296  "<html>\n<head>\n<title>%s</title>\n"
297  "<meta http-equiv=\"Content-Type\" "
298  "content=\"text/html; "
299  "charset=ISO-8859-1\">\n"
300  "<base href=\"%s\"/>\n</head>\n"
301  "<style type=\"text/css\">\n"
302  "img { border: 0; padding: 0 2px; vertical-align: "
303  "text-bottom; }\n"
304  "td { font-family: monospace; padding: 2px 3px; "
305  "text-align: right; vertical-align: bottom; }\n"
306  "td:first-child { text-align: left; padding: "
307  "2px 10px 2px 3px; }\n"
308  "table { border: 0; }\n"
309  "a.symlink { font-style: italic; }\n"
310  "</style>\n<body>\n"
311  "<h1>Index of %s</h1>\n"
312  "<hr/><table>\n", url_no_pw, curl->url, url_no_pw
313  );
314  }
315  }
316  else
317  {
318  log("Target is a directory.\n");
319  seterr("target is a directory");
320  doneurl();
321  return 0;
322  }
323  }
324  else if (!curl->putstream)
325  print("RETR %s\r\n", curl->url.getfile());
326  else
327  {
328  if (curl->create_dirs)
329  {
330  print("CWD %s\r\n", getdirname(curl->url.getfile()));
331  line = get_important_line();
332  if (!line)
333  return 0;
334  if (strncmp(line, "250", 3))
335  {
336  log("Path doesn't exist; creating directories...\n");
337  // create missing directories.
338  WvString current_dir("");
339  WvStringList dirs;
340  dirs.split(getdirname(curl->url.getfile()), "/");
341  WvStringList::Iter i(dirs);
342  for (i.rewind(); i.next(); )
343  {
344  current_dir.append(WvString("/%s", i()));
345  print("MKD %s\r\n", current_dir);
346  line = get_important_line();
347  if (!line)
348  return 0;
349  }
350  }
351  }
352  print("STOR %s\r\n", curl->url.getfile());
353  }
354 
355  log(WvLog::Debug5, "Waiting for response to %s\n", curl->putstream ? "STOR" :
356  curl->is_dir ? "LIST" : "RETR");
357  line = get_important_line();
358 
359  if (!line)
360  doneurl();
361  else if (strncmp(line, "150", 3))
362  {
363  log("Strange response to %s command: %s\n",
364  curl->putstream ? "STOR" : "RETR", line);
365  seterr(WvString("strange response to %s command",
366  curl->putstream ? "STOR" : "RETR"));
367  doneurl();
368  }
369 
370  }
371 
372  if (curl)
373  {
374  if (curl->is_dir)
375  {
376  line = data->blocking_getline(-1);
377  if (line && curl->outstream)
378  {
379  WvString output_line(parse_for_links(line.edit()));
380  if (!!output_line)
381  curl->outstream->write(output_line);
382  else
383  curl->outstream->write("Unknown format of LIST "
384  "response\n");
385  }
386  }
387  else
388  {
389  char buf[1024];
390 
391  if (curl->putstream)
392  {
393  while (curl->putstream->isreadable())
394  {
395  int len = curl->putstream->read(buf, sizeof(buf));
396  log(WvLog::Debug5, "Read %s bytes.\n%s\n", len, hexdump_buffer(buf, len));
397 
398  if (len)
399  {
400  int wrote = data->write(buf, len);
401  log(WvLog::Debug5,"Wrote %s bytes\n", wrote);
402  data->flush(0);
403  }
404  }
405  curl->putstream->close();
406  }
407  else
408  {
409  while (data->isreadable() && curl->outstream->isok())
410  {
411  int len = data->read(buf, sizeof(buf));
412  log(WvLog::Debug5, "Read %s bytes from remote.\n", len);
413 
414  if (len && curl->outstream)
415  {
416  int wrote = curl->outstream->write(buf, len);
417  log(WvLog::Debug5, "Wrote %s bytes to local.\n", wrote);
418  }
419  }
420  }
421  }
422 
423  if (!data->isok() || (curl->putstream && !curl->putstream->isok()))
424  {
425  log("OK, we should have finished writing!\n");
426  if (curl->putstream && data->isok())
427  data->close();
428  line = get_important_line();
429  if (!line)
430  {
431  doneurl();
432  return 0;
433  }
434 
435  if (strncmp(line, "226", 3))
436  log("Unexpected message: %s\n", line);
437 
438  if (curl->is_dir)
439  {
440  if (curl->outstream)
441  curl->outstream->write("</table><hr/></body>\n"
442  "</html>\n");
443  write("CWD /\r\n");
444  log(WvLog::Debug5, "Waiting for response to CWD /\n");
445  line = get_important_line();
446  if (!line)
447  return 0;
448 
449  if (strncmp(line, "250", 3))
450  log("Strange resonse to \"CWD /\": %s\n", line);
451  // Don't bother failing here.
452  }
453  doneurl();
454  }
455  else
456  {
457  log("Why are we here??\n");
458  }
459  }
460 
461  return 0;
462 }
463 
464 
466 {
467  real_execute(0);
468 }
469 
470 
471 WvString WvFtpStream::parse_for_links(char *line)
472 {
473  WvString output_line("");
474  trim_string(line);
475 
476  if (curl->is_dir && curl->outstream)
477  {
478  struct ftpparse fp;
479  int res = ftpparse(&fp, line, strlen(line));
480  if (res)
481  {
482  char *linkname = (char *)alloca(fp.namelen+1);
483  int i;
484  for (i = 0; i < fp.namelen; i++)
485  {
486  if (fp.name[i] >= 32)
487  linkname[i] = fp.name[i];
488  else
489  {
490  linkname[i] = '?';
491  }
492  }
493  linkname[i] = 0;
494 
495  WvString linkurl(curl->url);
496  if (linkurl.cstr()[linkurl.len()-1] != '/')
497  linkurl.append("/");
498  linkurl.append(linkname);
499  WvUrlLink *link = new WvUrlLink(linkname, linkurl);
500  curl->outstream->links.append(link, true);
501 
502  output_line.append("<tr>\n");
503 
504  output_line.append(WvString(" <td>%s%s</td>\n", linkname,
505  fp.flagtrycwd ? "/" : ""));
506 
507  if (fp.flagtryretr)
508  {
509  if (!fp.sizetype)
510  output_line.append(" <td>? bytes</td>\n");
511  else
512  output_line.append(WvString(" <td>%s bytes</td>\n",
513  fp.size));
514  if (fp.mtimetype > 0)
515  output_line.append(WvString(" <td>%s</td>\n", (fp.mtime)));
516  else
517  output_line.append(" <td>?</td>\n");
518  }
519  else
520  output_line.append(" <td></td>\n");
521 
522  output_line.append("</tr>\n");
523  }
524  }
525  return output_line;
526 }
527 
528 
529 WvIPPortAddr *WvFtpStream::parse_pasv_response(char *line)
530 {
531  if (strncmp(line, "227 ", 4))
532  {
533  log("Strange response to PASV command: %s\n", line);
534  seterr("strange response to PASV command");
535  return NULL;
536  }
537 
538  char *p = &line[3];
539  while (!isdigit(*p))
540  {
541  if (*p == '\0' || *p == '\r' || *p == '\n')
542  {
543  log("Couldn't parse PASV response: %s\n", line);
544  seterr("couldn't parse response to PASV command");
545  return NULL;
546  }
547  p++;
548  }
549  char *ipstart = p;
550 
551  for (int i = 0; i < 4; i++)
552  {
553  p = strchr(p, ',');
554  if (!p)
555  {
556  log("Couldn't parse PASV IP: %s\n", line);
557  seterr("couldn't parse PASV IP");
558  return NULL;
559  }
560  *p = '.';
561  }
562  *p = '\0';
563  WvString pasvip(ipstart);
564  p++;
565  int pasvport;
566  pasvport = atoi(p)*256;
567  p = strchr(p, ',');
568  if (!p)
569  {
570  log("Couldn't parse PASV IP port: %s\n", line);
571  seterr("couldn't parse PASV IP port");
572  return NULL;
573  }
574  pasvport += atoi(++p);
575 
576  WvIPPortAddr *res = new WvIPPortAddr(pasvip.cstr(), pasvport);
577 
578  return res;
579 }
WvString::edit
char * edit()
make the string editable, and return a non-const (char*)
Definition: wvstring.h:397
WvFtpStream::close
virtual void close()
Close this stream.
Definition: wvftpstream.cc:93
WvStream::write
virtual size_t write(const void *buf, size_t count)
Write data to the stream.
Definition: wvstream.cc:532
WvFtpStream::execute
virtual void execute()
The callback() function calls execute(), and then calls the user- specified callback if one is define...
Definition: wvftpstream.cc:465
WvStream::blocking_getline
char * blocking_getline(time_t wait_msec, int separator='\n', int readahead=1024)
This is a version of getline() that allows you to block for more data to arrive.
Definition: wvstream.cc:602
WvStreamClone::execute
virtual void execute()
The callback() function calls execute(), and then calls the user- specified callback if one is define...
Definition: wvstreamclone.cc:272
WvStream::flush
virtual bool flush(time_t msec_timeout)
flush the output buffer, if we can do it without delaying more than msec_timeout milliseconds at a ti...
Definition: wvstream.cc:707
WvStreamClone::close
virtual void close()
Close this stream.
Definition: wvstreamclone.cc:83
WvTCPConn::isok
virtual bool isok() const
Is this connection OK? Note: isok() will always be true if !resolved, even though fd==-1.
Definition: wvtcp.cc:375
WvErrorBase::strerror
static WvString strerror(int errnum)
A replacement for the operating system ::strerror() function that can map more kinds of error strings...
Definition: wverror.cc:91
WvStream::alarm_was_ticking
bool alarm_was_ticking
This will be true during callback execution if the callback was triggered by the alarm going off.
Definition: wvstream.h:54
trim_string
char * trim_string(char *string)
Trims whitespace from the beginning and end of the character string, including carriage return / line...
Definition: strutils.cc:59
WvStream::close
virtual void close()
Close the stream if it is open; isok() becomes false from now on.
Definition: wvstream.cc:341
WvStream::seterr
virtual void seterr(int _errnum)
Override seterr() from WvError so that it auto-closes the stream.
Definition: wvstream.cc:451
WvBufStream::isok
virtual bool isok() const
return true if the stream is actually usable right now
Definition: wvbufstream.cc:48
WvTCPConn
WvTCPConn tries to make all outgoing connections asynchronously (in the background).
Definition: wvtcp.h:39
WvString
WvString is an implementation of a simple and efficient printable-string class.
Definition: wvstring.h:329
WvFtpStream::pre_select
virtual void pre_select(SelectInfo &si)
pre_select() sets up for eventually calling ::select().
Definition: wvftpstream.cc:133
WvStream::isreadable
virtual bool isreadable()
Returns true if the stream is readable.
Definition: wvstream.cc:590
IWvStream::SelectInfo
the data structure used by pre_select()/post_select() and internally by select().
Definition: iwvstream.h:50
WvIPPortAddr
An IP+Port address also includes a port number, with the resulting form www.xxx.yyy....
Definition: wvaddr.h:393
WvFtpStream::post_select
virtual bool post_select(SelectInfo &si)
post_select() is called after ::select(), and returns true if this object is now ready.
Definition: wvftpstream.cc:149
WvStream::post_select
virtual bool post_select(SelectInfo &si)
post_select() is called after ::select(), and returns true if this object is now ready.
Definition: wvstream.cc:875
ftpparse
Definition: ftpparse.h:21
WvStream::read
virtual size_t read(void *buf, size_t count)
read a data block on the stream.
Definition: wvstream.cc:490
WvStream::alarm
void alarm(time_t msec_timeout)
set an alarm, ie.
Definition: wvstream.cc:1048
WvStream::isok
virtual bool isok() const
return true if the stream is actually usable right now
Definition: wvstream.cc:445
WvTCPConn::pre_select
virtual void pre_select(SelectInfo &si)
override pre_select() to cause select() results when resolving names.
Definition: wvtcp.cc:291
WvStreamClone::pre_select
virtual void pre_select(SelectInfo &si)
pre_select() sets up for eventually calling ::select().
Definition: wvstreamclone.cc:199
WvUrlStream
Definition: wvhttppool.h:104
WvStreamClone::geterr
virtual int geterr() const
If isok() is false, return the system error number corresponding to the error, -1 for a special error...
Definition: wvstreamclone.cc:149
WvFtpStream
Definition: wvhttppool.h:206
IWvStream::SelectRequest
A SelectRequest is a convenient way to remember what we want to do to a particular stream: read from ...
Definition: iwvstream.h:34
WvFdStream::close
virtual void close()
Closes the file descriptors.
Definition: wvfdstream.cc:117
WvStreamClone::isok
virtual bool isok() const
return true if the stream is actually usable right now
Definition: wvstreamclone.cc:136
hexdump_buffer
WvString hexdump_buffer(const void *buf, size_t len, bool charRep=true)
Produce a hexadecimal dump of the data buffer in 'buf' of length 'len'.
Definition: strutils.cc:245
WvTCPConn::post_select
virtual bool post_select(SelectInfo &si)
override post_select() to set the 'connected' variable as soon as we are connected.
Definition: wvtcp.cc:320
WvStringList
This is a WvList of WvStrings, and is a really handy way to parse strings.
Definition: wvstringlist.h:27
WvStringList::split
void split(WvStringParm s, const char *splitchars=" \t\r\n", int limit=0)
split s and form a list ignoring splitchars (except at beginning and end) ie.
Definition: wvstringlist.cc:19
WvStream::pre_select
virtual void pre_select(SelectInfo &si)
pre_select() sets up for eventually calling ::select().
Definition: wvstream.cc:844
WvStreamClone::post_select
virtual bool post_select(SelectInfo &si)
post_select() is called after ::select(), and returns true if this object is now ready.
Definition: wvstreamclone.cc:222
WvUrlRequest
Definition: wvhttppool.h:43