Changeset 380

Timestamp:

06/21/09 12:09:36 (11 years ago)

Author:

kovell

Message:

Feedfetch 1.7 external id support. Reduces potential load problems. Feed fetcher is identified.

Location:

dev/common

Files:

• admin/admin_feedsyndication.php (modified) (6 diffs)
• admin/feed_fetcher.php (modified) (11 diffs)
• feed.php (modified) (8 diffs)
• includes/class.killlist.php (modified) (6 diffs)

dev/common/admin/admin_feedsyndication.php

@@ -1 +1 @@
 <?php
-// ////
-// ////  //////
-// ////  liq's feed syndication mod v1.6
-// Kovell: last kill function activated and current year = 2008 changed in 1.6
-// //////////
-// //////////
-// //////////
-// //////////
+/*
+ * EDK Feed Syndication v1.7
+ * based on liq's feed syndication mod v1.5
+ *
+ */
+
 // set this to 1 if you are running a master killboard and want
 // to even fetch mails not related to your corp / alliance
-define(MASTER, 0);
-
-$version = "v1.6";
+define('MASTER', 0);
+
 @set_time_limit(0);
 require_once('feed_fetcher.php');
@@ -19 +16 @@
 require_once('common/includes/class.alliance.php');
 
-$page = new Page("Administration - Feed Syndication " . $version);
+$page = new Page("Administration - Feed Syndication " . $feedversion);
 $page->setCachable(false);
 $page->setAdmin();
@@ -66 +63 @@
                 $friends = "on";
             else $friends = "";
+            if ($_POST['apikills'] && in_array ($i, $_POST['apikills']))
+                $apikills = "on";
+            else $friends = "";
             $fstr = config::get('fetch_url_' . $i);
             $ftmp = explode(':::', $fstr);
-            // reset the feed lastkill details if the URL or friends has changed
-            if($_POST[$url] != $ftmp[0] || $friends != $ftmp[2] || ($_POST['newkills'] && !in_array ($i, $_POST['newkills'])) )
-                config::set($url, $_POST[$url] . ':::' . 0 . ':::' . $friends);
+            // reset the feed lastkill details if the URL, friends or api status has changed
+            if($_POST[$url] != $ftmp[0] || $friends != $ftmp[2] || $apikills != $ftmp[3] ||
+                                        ($_POST['newkills'] && !in_array ($i, $_POST['newkills'])) )
+                config::set($url, $_POST[$url] . ':::' . 0 . ':::' . $friends . ':::' . $apikills);
         }
         else
@@ -87 +88 @@
     if ($tmp[2] == "on")
         $friend[$i] = $tmp[2];
+        if ($tmp[3] == "on")
+        $apikills[$i] = $tmp[3];
 }
 // building the request query and fetching of the feeds
@@ -112 +115 @@
             if ($friend[$i])
                 $str .= '&friend=1';
+            if ($apikills[$i])
+                $str .= '&apikills=1';
             if ($_POST['fetch_losses'])
                 $str .= "&losses=1";
@@ -167 +172 @@
     $html .= "><b>New kills only?</b><br>";
 
+    $html .= "<td><input type=checkbox name=apikills[] id=apikills value=" . $i;
+    if ($apikills[$i])
+        $html .= " checked=\"checked\"";
+    $html .= "><b>API verified only?</b><br>";
+
     $html .= "<td><input type=checkbox name=fetch_feed[] id=fetch value=" . $i;
     if ($feed[$i])

dev/common/admin/feed_fetcher.php

@@ -1 +1 @@
 <?php
-//////
-//////  //////
-//////  liq's feed syndication mod v1.6
-// Kovell: last kill function activated in 1.6
-////////////
-// Fetcher class used with common/feed.php on a remote board to fetch kills
-// with a given pilot, corp or alliance
-
-
-// Fetches $this->lastkllid_ = last kill in feed
-// $this->finalkllid_ = last kill on board including kills not in feed
+/*
+ * EDK Feed Syndication v1.7
+ * based on liq's feed syndication mod v1.5
+ *
+ */
 
 @set_time_limit(0);
 @ini_set('memory_limit',999999999);
+$feedversion = "v1.7";
 
 require_once( "common/includes/class.kill.php" );
 require_once( "common/includes/class.parser.php" );
-if ( file_exists("common/includes/class.comments.php") ) // for the Eve-Dev Comment Class
-        require_once( "common/includes/class.comments.php" );
-if ( file_exists("../../common/class.comment.php") ) // for the D2 Killboard Comment Class
-        require_once( "common/class.comment.php" );
+require_once( "common/includes/class.comments.php" );
 
 $insideitem = false;
@@ -61 +53 @@
         function grab($url, $str, $trackfriend = '', $trackkey = '')
         {
-                global $x, $uurl;
+                global $x, $uurl, $feedversion;
                 //                $this->trackurl_ = $trackurl;
                 $this->trackfriend_ = $trackfriend;
                 $this->trackkey_ = $trackkey;
                 $x=0;
-                $fetchurl = $url.$str;
+                $fetchurl = $url.$str."&board=".urlencode(KB_TITLE);
+                if(!strpos($fetchurl,'?')) $fetchurl =
+                                substr_replace($fetchurl,'?', strpos($fetchurl,'&'),0);
                 $uurl = $url;
                 $xml_parser = xml_parser_create("UTF-8");
@@ -76 +70 @@
 
                 $http = new http_request($fetchurl);
+                $http->set_useragent("EDK Feedfetcher ".$feedversion);
                 $http->set_timeout(120);
-                $http->set_cookie('D2', 'a%3A4%3A%7Bi%3A0%3Bs%3A4%3A%221456%22%3Bi%3A1%3Bs%3A40%3A%22ca313afe9822b1d172ce660758e915fe81810780%22%3Bi%3A2%3Bi%3A1359068654%3Bi%3A3%3Bi%3A2%3B%7D');
                 $http->set_cookie('PHPSESSID', 'a2bb4a7485eaba91b9d8db6aafd8ec5d');
-                #Cookie: D2=; PHPSESSID=a2bb4a7485eaba91b9d8db6aafd8ec5d
                 $data = $http->get_content();
-                $data = trim(preg_replace('<<!--.*?-->>', '', $data)); // remove <!-- Cached --> message, else it will break gzinflate
-
+//              $data = trim(preg_replace('<<!--.*?-->>', '', $data)); // remove <!-- Cached --> message, else it will break gzinflate
+                $data = preg_replace('<<!--.*?-->>', '', $data); // remove <!-- Cached --> message, else it will break gzinflate
                 if (!@gzinflate($data))
                 {
@@ -116 +109 @@
         function startElement($parser, $name, $attrs)
         {
-                global $insideitem, $tag, $title, $description, $link;
+                global $insideitem, $tag, $title, $description, $link, $apiID;
                 //      if ($insideitem)
                 $tag = $name;
@@ -127 +120 @@
         function endElement($parser, $name)
         {
-                global $insideitem, $tag, $title, $description, $link, $html, $x, $uurl;
+                global $insideitem, $tag, $title, $description, $link, $html, $x, $uurl, $apiID;
 
                 if ($name == "ITEM")
@@ -133 +126 @@
                         if ( isset( $description ) )
                         {
-                                $parser = new Parser( $description );
-                                $killid = $parser->parse( true );
+                                // If the kill has an external id then check if it is already
+                                // on this board.
+                                $apiID = intval($apiID);
+                                if( $apiID)
+                                {
+                                        $qry = new DBQuery();
+                                        $qry->execute("SELECT 1 FROM kb3_kills WHERE kll_external_id = ".$apiID);
+                                        if(!$qry->recordCount())
+                                        {
+                                                $parser = new Parser( $description );
+                                                $killid = $parser->parse( true );
+                                        }
+                                        else
+                                        {
+                                                $killid = -3;
+                                                $html .= "Killmail already posted <a href=\"?a=kill_detail&amp;kll_external_id=".$apiID."\">here</a>.<br>";
+                                        }
+                                }
+                                else
+                                {
+                                        $parser = new Parser( $description );
+                                        $killid = $parser->parse( true );
+                                }
                                 if($this->trackkey_ != '' && $this->tracklast_ < intval($title) && intval($title) > 0)
                                 {
                                         config::set($this->trackkey_, $uurl . ':::' . intval($title) . ':::' . $this->trackfriend_);
                                 }
-                                if ( $killid == 0 || $killid == -1 || $killid == -2 )
+                                if ( $killid == 0 || $killid == -1 || $killid == -2 || $killid == -3)
                                 {
                                         if ( $killid == 0 && config::get('fetch_verbose') )
@@ -146 +160 @@
                                                 $html .= "Killmail is not related to ".KB_TITLE.".<br>";
                                         if ( $killid == -1 && config::get('fetch_verbose') )
-                                                $html .= "Killmail already posted <a href=\"?a=kill_detail&kll_id=".$parser->dupeid_."\">here</a>.<br>";
+                                                $html .= "Killmail already posted <a href=\"?a=kill_detail&amp;kll_id=".$parser->dupeid_."\">here</a>.<br>";
                                 }
                                 else
                                 {
-
                                         $qry = new DBQuery();
                                         $qry->execute( "insert into kb3_log (log_kll_id, log_site, log_ip_address, log_timestamp) values( ".
@@ -156 +169 @@
                                         $html .= "Killmail successfully posted <a href=\"?a=kill_detail&kll_id=".$killid."\">here</a>.<br>";
 
-                                        if (class_exists('Comments') && config::get('fetch_comment'))
-                                        { // for the Eve-Dev Comment Class
+                                        if (config::get('fetch_comment'))
+                                        { 
                                                 $comments = new Comments($killid);
                                                 $comments->addComment("Feed Syndication", config::get('fetch_comment')." mail fetched from: ".$uurl.")");
-                                        }
-                                        if (class_exists('Comment') && config::get('fetch_comment'))
-                                        { // for the D2 Killboard Comment Class
-                                                $comment = new Comment($killid);
-                                                $comment->postComment(config::get('fetch_comment')." \n\n\n <i>mail fetched from:\n ".$uurl."</i>", "liquidism");
                                         }
                                         $x++;
@@ -174 +182 @@
                         $link = "";
                         $insideitem = false;
+                        $apiID = false;
                 }
         }
@@ -179 +188 @@
         function characterData($parser, $data)
         {
-                global $insideitem, $tag, $title, $description, $link;
+                global $insideitem, $tag, $title, $description, $link, $apiID;
                 if ($insideitem)
                 {
@@ -193 +202 @@
                                         $link .= $data;
                                         break;
+                                case "APIID":
+                                        $apiID .= $data;
                         }
                 }

dev/common/feed.php

@@ -2 +2 @@
 /*
  * Create a syndication feed of kills stored on this board.
+ *
+ * Flags
+ * week = week
+ * year = year
+ * lastkllid = return all kills lastkllid on (ordered by kll_id)
+ * range = return all kills with lastkllid <= id <= lastkllid + range
+ * APIkills = restrict results to kills with an external id set
+ * pilot = pilot to retrieve kills for
+ * corp = corp_name = corp to retrieve kills for
+ * alli = alli_name = alliance to retrieve kills for
+ * master = retrieve all kills
+ * friend = set pilot/corp/alli as involved killer (default is victim)
+ * combined = return both kills and losses
+ *
  */
 @set_time_limit(120);
@@ -12 +26 @@
 
 $config = new Config(KB_SITE);
-$qry = new DBQuery();
-if(config::get('kill_classified'))
-{
-        $qry->execute('SELECT max(kll_id) as finalkill FROM kb3_kills WHERE kll_timestamp < "'.(date('Y-m-d H:i:s',time()-config::get('kill_classified')*60*60)).'"');
-}
-else $qry->execute('SELECT max(kll_id) as finalkill FROM kb3_kills');
-$row=$qry->getRow();
-$finalkill = intval($row['finalkill']);
+// maximum amount of kills to return.
+$maxreturned = 100;
 $html .= '<?xml version="1.0" encoding="UTF-8" ?>
 <rss version="2.0">
@@ -26 +34 @@
         <link>'.KB_HOST.'</link>
         <description>Kill Feed v1.6</description>
-        <copyright>'.KB_TITLE.'</copyright>
-        <finalkill>'.$finalkill.'</finalkill>\n';
-if($_GET['combined']) $html .= '<combined>true</combined>\n';
+        <copyright>'.KB_TITLE."</copyright>\n";
+if($_GET['combined']) $html .= "<combined>true</combined>\n";
 $klist = new KillList();
 $klist->setPodsNoobShips(true);
@@ -53 +60 @@
 
 $kid = intval($_GET['lastkllid']);
-if ($kid && method_exists($klist, 'setMinKllID'))
+if ($kid)
 {
         $klist->setMinKllID($kid);
         $klist->setOrderBy(' kll.kll_id ASC');
         $klist->setOrdered(true);
-}
-
+        $klist->setLimit($maxreturned);
+        if(intval($_GET['range'])) $klist->setMaxKllID(intval($_GET['range'])+$kid);
+}
+// If asked, set feed to only retrieve kills with an external id set.
+if (intval($_GET['APIkills'])) $klist->setAPIKill();
 if ($_GET['pilot'] || $_GET['pilot_name'])
 {
@@ -193 +203 @@
 
 $kills = array();
+$finalkill = 0;
 while ($kill = $klist->getKill())
 {
@@ -199 +210 @@
                 continue;
         }
-
+        if($finalkill < $kill->getID())$finalkill = $kill->getID();
         $kills[$kill->getID()] = $kill->getTimestamp();
 }
@@ -206 +217 @@
         asort($kills);
 }
+$qry = new DBQuery();
+// If kills returned = $maxreturned assume that it was limited and set
+// last kill as the lower of highest kill id returned or highest non-classified
+// kill
+if($klist->getCount() != $maxreturned)
+{
+        $qry = new DBQuery();
+        if(config::get('kill_classified'))
+        {
+                $qry->execute('SELECT max(kll_id) as finalkill FROM kb3_kills WHERE kll_timestamp < "'.(date('Y-m-d H:i:s',time()-config::get('kill_classified')*60*60)).'"');
+        }
+        else $qry->execute('SELECT max(kll_id) as finalkill FROM kb3_kills');
+        $row=$qry->getRow();
+        $finalkill = intval($row['finalkill']);
+}
+elseif(config::get('kill_classified'))
+{
+        // Check if there are classified kills with lower kill ids still to come.
+        $qry->execute('SELECT max(kll_id) as finalkill FROM kb3_kills WHERE kll_timestamp < "'.(date('Y-m-d H:i:s',time()-config::get('kill_classified')*60*60)).'"');
+        $row=$qry->getRow();
+        if($finalkill > intval($row['finalkill'])) $finalkill = intval($row['finalkill']);
+}
+
+$html .= '<finalkill>'.$finalkill.'</finalkill>\n';
 foreach ($kills as $id => $timestamp)
 {
@@ -213 +248 @@
                                 <description><![CDATA[ '.$kill->getRawMail().' ]]></description>
                                 <guid>?a=kill_detail&amp;kll_id='.$id.'</guid>
-                                <pubDate>'.strftime("%a, %d %b %Y %T %Z", strtotime($timestamp))."</pubDate>
-                                </item>\n";
+                                <pubDate>'.strftime("%a, %d %b %Y %T %Z", strtotime($timestamp))."</pubDate>\n";
+        if($kill->getExternalID()) $html .= "<apiID>".$kill->getExternalID()."</apiID>\n";
+        $html .= '</item>\n';
 }
 $html .= '</channel></rss>';

dev/common/includes/class.killlist.php

@@ -70 +70 @@
                 else
                                 {
-                                        $this->sql_ .= 'select kll.kll_id, kll.kll_timestamp, plt.plt_name, 
-                                                                crp.crp_name, crp.crp_id, ali.all_name, ali.all_id,
-                                kll.kll_system_id, kll.kll_ship_id,
-                                kll.kll_victim_id, plt.plt_externalid,
-                                kll.kll_crp_id, kll.kll_points, kll.kll_isk_loss,
-                                        shp.shp_class, shp.shp_name,
-                                        shp.shp_externalid, shp.shp_id,
-                                                scl.scl_id, scl.scl_class, scl.scl_value,
-                                                sys.sys_name, sys.sys_sec,
-                                fbplt.plt_name as fbplt_name,
-                                fbplt.plt_externalid as fbplt_externalid,
-                                fbcrp.crp_name as fbcrp_name';
+                                        $this->sql_ .= 'select kll.kll_id, kll.kll_timestamp, kll.kll_external_id,
+                                                                plt.plt_name, crp.crp_name, crp.crp_id,
+                                                                ali.all_name, ali.all_id,
+                                                                kll.kll_system_id, kll.kll_ship_id,
+                                                                kll.kll_victim_id, plt.plt_externalid,
+                                                                kll.kll_crp_id, kll.kll_points, kll.kll_isk_loss,
+                                                                shp.shp_class, shp.shp_name,
+                                                                shp.shp_externalid, shp.shp_id,
+                                                                scl.scl_id, scl.scl_class, scl.scl_value,
+                                                                sys.sys_name, sys.sys_sec,
+                                                                fbplt.plt_name as fbplt_name,
+                                                                fbplt.plt_externalid as fbplt_externalid,
+                                                                fbcrp.crp_name as fbcrp_name';
                                 }
             }
@@ -157 +158 @@
                         if($this->apikill_)
                         {
-                $this->sql_ .= $sqlwhereop." AND kll.kll_external_id IS NOT NULL ";
+                $this->sql_ .= $sqlwhereop." kll.kll_external_id IS NOT NULL ";
                 $sqlwhereop = ' AND ';
                         }
@@ -206 +207 @@
             if ($this->minkllid_)
             {
-                $this->sql_ .= $sqlwhereop.'kll.kll_id > '.$this->minkllid_.' ';
+                $this->sql_ .= $sqlwhereop.'kll.kll_id >= '.$this->minkllid_.' ';
+                $sqlwhereop = ' AND ';
+            }
+
+            // Get all kills before given kill id (used for feed syndication)
+            if ($this->maxkllid_)
+            {
+                $this->sql_ .= $sqlwhereop.'kll.kll_id <= '.$this->maxkllid_.' ';
                 $sqlwhereop = ' AND ';
             }
@@ -390 +398 @@
             $kill->setFBCorpName($row['fbcrp_name']);
             $kill->setKillPoints($row['kll_points']);
-            $kill->iskloss_ = $row['kll_isk_loss'];
+                        $kill->setExternalID($row['kll_external_id']);
+                        $kill->setISKLoss($row['kll_isk_loss']);
                         $kill->plt_ext_ = $row['plt_externalid'];
             $kill->fbplt_ext_ = $row['fbplt_externalid'];
@@ -631 +640 @@
         function setAPIKill($hasid = true)
         {
-                $this->extid_ = $hasid;
+                $this->apikill_ = $hasid;
         }
 
@@ -641 +650 @@
     function setMinKllID($id)
     {
-        $this->timeframe_ = '';
         $this->minkllid_ = $id;
+    }
+
+    function setMaxKllID($id)
+    {
+        $this->maxkllid_ = $id;
     }