I'm sorry the code is a bit of a mess as I have been hacking at it -- adding options at random while exploring curl's features. Maybe you can see something there that I am missing.
Some odd things are if the curl STDERR file is type "w" it seems to write over itself when it switches from http to https. Also I can't access the cookie file in any way while the connection is open, so I had to request the header and parse it to get at some of the data. And I removed the javascript stuff that fetchs a few tokens with ajax because that doesn't really matter for this problem. The very last call to getcurl is the one where I get no chunks or data in response. This site is pretty ajax heavy, so I'm not expecting much other than some html with code I have to parse to fetch more data. The site is also not well designed for mal-formed queries, but it usually sends some kind of html response.
There are a lot of 302 redirects that curl chases down fantastically. Perhaps the key to my problem is that I can't get curl to consistently switch between HEAD and GET requests. By the time curl closes $process there are 3 connections to shut down and maybe they all aren't configured the same way due to the redirects where it is opening other connections. I don't know.
I didn't post the debug.txt log file because it would take a while to sanitize, but I can if you want to see that too.
Code: Select all
$postdata = array(
"username" => "username",
"password" => "password",
"rememberMe" => "true",
"lt" => "e1s1",
"_eventId" => "submit",
"submit" => "Login"
);
$postfield = "";
foreach ($postdata as $key => $value) {
$postfield.=$key . "=" . $value . "&";
}
unlink("debug.txt"); //remove previous debug file
$debug=fopen("debug.txt","a"); // has to be append or cURL will overwrite
$cookie_file = 'cookiejar.txt';
$process = curl_init();
clrcookie($process); // reset cookie session
$url = 'http://domain.com/service-portal';
$results=(getcurl($process,$url,1, $debug,$cookie_file,"1"));
//echo nl2br($results);
// get id from header
preg_match_all("/JSESSIONID=(.*?);/ms", $results, $match);
$ssid=trim($match[1][count($match)-1]); //get last or only id
//echo "<br>ssid=".$ssid."<br>";
$results=getcurl($process,"https://sub.domain.com/cas/login?service=http%3A%2F%2Fdomain.com%2Fservice-portal%2Fhome",1, $debug,$cookie_file,"1");
//echo nl2br($results);
//copy($cookie_file,'cookie.txt'); // tried to make local copy for access to cookies...not working
// REMOVED JAVASCRIPT stuff to generate parts of next URL to login
$url = "https://sub.domain.com/cas/login;jsessionid=" . $ssid . "?service=http%3A%2F%2Fdomain.com%2Fservice-portal%2Fhome";
$results=post($process,$url, $postfield, $debug,$cookie_file,"https://sub.domain.com/cas/login?service=http%3A%2F%2Fdomain.com%2Fservice-portal%2Fhome");
// LOGIN successful, fetch a page
$url="http://sub2.domain.com/sold/listing/cache/sb_sold_results.jsp?slim=sold&cit=true&sm=3&searchPage=%2Fsold%2Flisting%2Fcache%2Fsb_search_page2.jsp&robw=&is=&type=&fromSDint=&fromSDmon=1&fromSDyr=&toSDmon=12&toSDyr=&fromLength=35&toLength=35&luom=126&fromYear=2001&toYear=2006&fromPrice=&toPrice=¤cyid=100&hmid=0&ftid=0&enid=0&city=&cint=&spid=&rid=&ywbid=&ps=30&bn=&includeSoldComments=";
$results=(getcurl($process,$url,0, $debug,$cookie_file,"1"));
echo $results;
curl_close($process);
function clrcookie($process){
curl_setopt($process,CURLOPT_COOKIESESSION,1);
}
function getcurl($process,$url, $head_only=1,$debug=NULL, $cookies=NULL, $refer="1") {
$headers[] = "Accept[text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8]";
$headers[] = "Accept-Charset[ISO-8859-1,utf-8;q=0.7,*;q=0.7]";
$headers[] = "Keep-Alive[115]";
$headers[] = "Connection: Keep-Alive";
$headers[] = "Content-type: application/x-www-form-urlencoded";
$user_agent = "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.17) Gecko/20110422 Ubuntu/10.10 (maverick) Firefox/3.6.17";
curl_setopt($process, CURLOPT_FOLLOWLOCATION, 20);
curl_setopt($process, CURLOPT_NOBODY,1); // only headers
if ($refer == "1")
curl_setopt($process, CURLOPT_AUTOREFERER, 1);
else
curl_setopt($process, CURLOPT_REFERER, $refer);
curl_setopt($process, CURLOPT_URL, $url);
curl_setopt($process, CURLOPT_HTTPHEADER, $headers);
curl_setopt($process, CURLOPT_USERAGENT, $user_agent);
if ($cookies != NULL) {
curl_setopt($process, CURLOPT_COOKIEFILE, $cookies);
curl_setopt($process, CURLOPT_COOKIEJAR, $cookies);
}
curl_setopt($process, CURLOPT_TIMEOUT, 30);
curl_setopt($process, CURLOPT_RETURNTRANSFER, 1);
if ($debug != NULL) {
curl_setopt($process, CURLOPT_STDERR, $debug);
curl_setopt($process, CURLOPT_VERBOSE, TRUE);
}
if($head_only==1) curl_setopt($process, CURLOPT_HEADER, true); // header will be at output
else
curl_setopt($process, CURLOPT_HEADER, false);
$return = curl_exec($process);
return $return;
}
function post($process,$url, $data, $debug=NULL,$cookies=NULL,$refer="1") {
curl_setopt($process,CURLOPT_URL,$url);
$headers[] = "Accept[text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8]";
$headers[] = "Connection: Keep-Alive";
$headers[] = "Content-type: application/x-www-form-urlencoded";
$user_agent = "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.17) Gecko/20110422 Ubuntu/10.10 (maverick) Firefox/3.6.17";
curl_setopt($process, CURLOPT_HEADER, FALSE); // header will be at output
curl_setopt($process, CURLOPT_FOLLOWLOCATION, 20);
if ($refer == "1")
curl_setopt($process, CURLOPT_AUTOREFERER, 1);
else
{
curl_setopt($process, CURLOPT_AUTOREFERER, 0);
curl_setopt($process, CURLOPT_REFERER, $refer);
}
curl_setopt($process, CURLOPT_HTTPHEADER, $headers);
curl_setopt($process, CURLOPT_USERAGENT, $user_agent);
if ($cookies != NULL) {
curl_setopt($process, CURLOPT_COOKIEFILE, $cookies);
curl_setopt($process, CURLOPT_COOKIEJAR, $cookies);
}
curl_setopt($process, CURLOPT_TIMEOUT, 30);
curl_setopt($process, CURLOPT_RETURNTRANSFER, 1);
if ($debug != NULL) {
curl_setopt($process, CURLOPT_STDERR, $debug);
curl_setopt($process, CURLOPT_VERBOSE, TRUE);
}
$return = curl_exec($process);
curl_setopt($process, CURLOPT_POSTFIELDS, $data);
curl_setopt($process, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($process, CURLOPT_FOLLOWLOCATION, 1);
curl_setopt($process, CURLOPT_POST, 1);
if ($debug != NULL) {
curl_setopt($process, CURLOPT_STDERR, $debug);
curl_setopt($process, CURLOPT_VERBOSE, 1);
}
$return = curl_exec($process);
return $return;
}