import java.io.*;
import java.sql.*;
import java.net.*;
import java.util.*;
import org.htmlparser.*;
import org.htmlparser.nodes.*;
import org.htmlparser.util.NodeList;
import org.htmlparser.util.ParserException;
import org.htmlparser.tags.LinkTag;
import org.htmlparser.visitors.TagFindingVisitor;
import org.apache.commons.httpclient.util.URIUtil;

public class RssExpander
{
	private static Settings settings = new Settings();

	private static final int FEED_INACTIVE = -1;
	private static final int FEED_ACTIVE = 1;

	private static final int ARTICLE_NORM = 0;
	private static final int ARTICLE_UPDATE = 1;
	private static final int ARTICLE_ERROR = 2;

	private static String sqlRssTable	= null;

    public static void main(String[] args) throws Exception
    {
		Connection conn = null;

		//Setup
	    {
	    	System.setErr( new Log4jPrintStream(System.err) );

			//Setup params
			{
				HashMap<String,String> cmds = new HashMap<String,String>();

				for(String envName : System.getenv().keySet())
				{
					cmds.put(envName,System.getenv(envName));
				}

				for (int i = 0; i<args.length; i++)
			    {
			    	String name = null;
			    	String value = null;

			    	if(args[i].charAt(0) == '-')
			    	{
			    		if(args[i].length()>1)
			    		{
			    			name = args[i].substring(1);
			    			value =	String.valueOf(true);
			    		}else Bomb("Invalid arg:" + args[i]);
			    	}else
			    	{
			    		if( i < args.length -1)
			    		{
			    			name = args[i];
			    			value = args[i+1];
			    			i++;
			    		}else Bomb("Expecting Data for:" + args[i]);
			    	}

			    	if(name!=null && value!=null)
						cmds.put(name,value);
			    }

				String argName = null;
				try
				{
					for (String keyName:cmds.keySet())
					{
						argName = keyName;
						settings.set(keyName,cmds.get(keyName));
					}
				}catch(Exception e)
				{
					Bomb("Invalid Argument Entry For " + argName);
				}
			}

			//Connect To Driver
			{
				Debug("Loading Driver");

				try
				{
					Class.forName("com.mysql.jdbc.Driver").newInstance();
				}catch(java.lang.ClassNotFoundException ex2)
				{
					Bomb("Could Not Find Driver");
				}catch(Exception e)
				{
					Bomb("Could Not Load Driver",e);
				}

				Debug("Connecting To Database");

				try
				{
					String url = "jdbc:mysql://%MYSQLHOST%:%MYSQLPORT%/%DATABASE%?user=%USER%&password=%PASS%";

					url = url.replaceAll("%MYSQLHOST%",settings.sqlHost);
					url = url.replaceAll("%MYSQLPORT%",String.valueOf(settings.sqlPort));
					url = url.replaceAll("%DATABASE%",settings.sqlDB);
					url = url.replaceAll("%USER%",settings.sqlUser);
					url = url.replaceAll("%PASS%",settings.sqlPass);

					conn = DriverManager.getConnection(url);
					conn.setAutoCommit(true);
				}catch(Exception e)
				{
					if(StackToStr(e,false).indexOf("Another instance of Derby may have already booted the database")>-1)
					{
						Bomb("Could Not Connect To Database It Is Already In Use");
					}else
						Bomb("Could Not Connect To Database",e);
				}
			}

			//Create Tables
			{
				try
				{
					//RssFeeds
					{
						String sql = 	"CREATE TABLE RssFeeds (" +
										"Id INT NOT NULL AUTO_INCREMENT," +
										"Title VARCHAR(1000) NOT NULL DEFAULT ''," +
										"Status INT NOT NULL DEFAULT " + FEED_ACTIVE + "," +
										"Link VARCHAR(1000) NOT NULL UNIQUE," +
										"Path VARCHAR(1000) NOT NULL UNIQUE DEFAULT 'index.rss'," +
										"SelfLink VARCHAR(1000) NOT NULL DEFAULT ''," +
										"Des VARCHAR(1000) NOT NULL DEFAULT ''," +
										"PubDate TIMESTAMP NOT NULL DEFAULT 0," +
										"Updated TIMESTAMP NOT NULL DEFAULT 0,"+
										"LastWrite TIMESTAMP NOT NULL DEFAULT 0,"+
										"PRIMARY KEY(Id)" +
										")";

						CreateTable(conn,"RssFeeds",sql);
					}

					//Feed0001
					{
					    sqlRssTable = 	"CREATE TABLE %FEED% (" +
					    				"Id INT NOT NULL AUTO_INCREMENT," +
										"Article INT NOT NULL," +
										"Title VARCHAR(1000) NOT NULL," +
										"Idx INT  NOT NULL DEFAULT -1," +
										"Status INT NOT NULL," +
										"Link VARCHAR(1000) NOT NULL," +	//Initial URL
										"ResUrl VARCHAR(1000) NOT NULL," + //Response URL
										"Guid VARCHAR(1000) NOT NULL UNIQUE," +
										"PubDate TIMESTAMP NOT NULL DEFAULT 0," +
										"Created TIMESTAMP NOT NULL DEFAULT 0," +
										"PRIMARY KEY(Id)" +
										")";
					}

					//Articles
					{
			        	String	sql = 	"CREATE TABLE Articles (" +
			        					"Id INT NOT NULL AUTO_INCREMENT," +
			        					"Article VARCHAR(32000) NOT NULL DEFAULT '',"+
			        					"ResUrl VARCHAR(400) NOT NULL UNIQUE," + //Response URL
										"Modified TIMESTAMP NOT NULL DEFAULT 0," +
										"Created TIMESTAMP NOT NULL DEFAULT 0," +
										"DisDate TIMESTAMP NOT NULL DEFAULT 0," +
										"PRIMARY KEY(Id)" +
			        					")";

						CreateTable(conn,"Articles",sql);
					}
				}catch(Exception e)
				{
					Bomb("Failed To Create Tables",e);
				}
			}

			System.getProperties().setProperty("sun.net.client.defaultConnectTimeout",String.valueOf(settings.requestTimeout));
	    }

	    //Get And Write Articles
	    {
	    	Debug("Starting Scrape");

	    	try
		    {
		        Statement stmtF = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY,ResultSet.CONCUR_UPDATABLE);
		        ResultSet resF = stmtF.executeQuery("SELECT * FROM RssFeeds WHERE Status=" + FEED_ACTIVE);

		        while(resF.next())
		        {
		        	boolean aUpdateHasOcurred = false;

		        	String title = resF.getString("Title") ;
		        	int id = resF.getInt("Id");
		        	String feedTable = GetFeedTableName(id);

	        		Log("Checking " + (Util.StrNullOrEmpty(title) ? (feedTable) : title));

					CreateTable(conn,feedTable,sqlRssTable.replaceFirst("%FEED%",feedTable));

					//Update\Add New Article
	        		{
						try
						{
		        			RssFeed rss = RssFeed.parse( new BufferedInputStream( GetStream(resF.getString("Link")) ) );
		        			RssItem[] items = rss.getItems();

						    Statement stmtClear = conn.createStatement();
						    stmtClear.executeUpdate("Update " + feedTable + " Set Idx=-1");
						    stmtClear.close();

							for (int i = 0; i<items.length && i<settings.MAX_NUM_OF_ITEMS; i++)
						    {
						    	try
						    	{
									boolean feedItemExists = false;

									//Update
									{
										PreparedStatement stmtFT = conn.prepareStatement("Select * From " + feedTable + " Where Guid=?",ResultSet.TYPE_FORWARD_ONLY,ResultSet.CONCUR_UPDATABLE);
										stmtFT.setString(1,items[i].guid);
										ResultSet resFT = stmtFT.executeQuery();

										if(resFT.next())
										{
											feedItemExists = true;

											boolean deleteAndGetNew = false;
											int articleId = resFT.getInt("Article");

											PreparedStatement stmtA = conn.prepareStatement("Select * From Articles Where Id=?",ResultSet.TYPE_FORWARD_ONLY,ResultSet.CONCUR_UPDATABLE);
											stmtA.setInt(1,articleId);
											ResultSet resA = stmtA.executeQuery();

											if(resA.next())
											{
												boolean updatePeriodHasPassed = false;

												long nextCheck,now;
												//Calc New Due Update
												{
													//Update Frequency
													int status = resFT.getInt("Status");

													long[] delay;
													long maxDelay;

													if( status == ARTICLE_NORM)
														delay = settings.delay_NORM;
													else if(status == ARTICLE_UPDATE)
														delay = settings.delay_UPDATE;
													else//if(status == ARTICLE_ERROR)
														delay = settings.delay_ERROR;

													maxDelay = delay.length > 0 ? delay[delay.length-1] : 0;

													long modified = resA.getTimestamp("Modified").getTime();
													long checked = resA.getTimestamp("Checked").getTime();
													now = GetCurrentTimestamp().getTime();

													nextCheck = modified;

													for(int j=0;j<delay.length && nextCheck<=checked;j++)
														nextCheck += delay[j];

													if(nextCheck<checked)
													{
														updatePeriodHasPassed = true;
														nextCheck = checked + maxDelay;
													}
												}

												if(nextCheck < now)
												{
													Log("\tUpdating Article " + String.format("%1$-5d",articleId) + " " + Pad(items[i].title,40) + "   " + items[i].link);

													String mainArtNew = null;
													NodeList nodeList = null;
													boolean usingRssDes = false;

													HttpRequest hr = null;
													try
												    {
												    	hr = HttpRequest.Create(items[i].link);
												        ExtractMainArticle.ExtractMainArticleNodes(GetPage(hr.getInputStream()),items[i].link,items[i].title);

												        nodeList = GetBestMatchingArticle(ExtractMainArticle.GetArticlesByPos(),items[i].content);
														mainArtNew = nodeList.toHtml();
														mainArtNew = Truncate(mainArtNew,32000);
												    }catch(FileNotFoundException exFNF)
												    {
												    	Log("\t\tCould Not Find Article");
												    }catch(Exception ex4)
												    {
												    	if(settings.debug)
												    		Log("\t\tFailed To Update Article ",ex4);
												    	else
												    		Log("\t\tFailed To Update Article " + ex4.getMessage());
												    }

													if(mainArtNew!=null && nodeList!=null)
													{
														int desLen = GetDescriptionRealLen(items[i].content);
														int artLen = GetArticleRealLen(nodeList.toNodeArray());

														if(desLen > artLen && false)
														{
															mainArtNew = Truncate(items[i].content,32000);
															usingRssDes = true;
														}
													}

												    ResultSet resA2;

											        PreparedStatement stmtQ = conn.prepareStatement("SELECT * FROM Articles WHERE ResUrl=?",ResultSet.TYPE_FORWARD_ONLY,ResultSet.CONCUR_UPDATABLE);	//Make sure that the new ResUrl doesn't already exist
											        stmtQ.setString(1,hr.getLastURL());
											        ResultSet resQ = stmtQ.executeQuery();

													if(resQ.next())
														resA2 = resQ;
													else
														resA2 = resA;

													if(mainArtNew!=null)
													{
												        String mainArtOld = resA2.getString("Article");

												        if(mainArtNew.length() < mainArtOld.length() * 0.1)
												        	Log("\t\tNew Article Is Too Small Ignoring");
												        else
												        {
												        	if(!mainArtOld.equals(mainArtNew))
												        	{
											        			Log("\t\tA New Version Of The Article Was Found");

																if(usingRssDes)
																	Log("\t\tUsing Rss Description");

																resA2.updateString("Article",mainArtNew);
																resA2.updateTimestamp("Modified",GetCurrentTimestamp());

																aUpdateHasOcurred = true;
												        	}

															if(updatePeriodHasPassed)
																resFT.updateInt("Status",ARTICLE_NORM);
															else
																resFT.updateInt("Status",GetArticleStatus(mainArtNew));
												        }
													}

											        resFT.updateString("ResUrl",hr.getLastURL());
									        		resA2.updateString("ResUrl",hr.getLastURL());
											        resA2.updateTimestamp("Checked",GetCurrentTimestamp());
											        resA2.updateRow();

													resA2.close();
											        stmtQ.close();
												}
											}else
											{
												Log("\tThis Should Not Have Ocurred, An Entry With No Article " + "Table=" + feedTable + " Article Id=" + articleId);
												deleteAndGetNew = true;
											}

											if(!deleteAndGetNew)
											{
												resFT.updateString("Title",items[i].title);
												resFT.updateString("Link",items[i].link);
												resFT.updateTimestamp("PubDate",new Timestamp(items[i].date.getTime()));
												resFT.updateInt("Idx",i);
												resFT.updateRow();
											}else
											{
												resFT.deleteRow();
												feedItemExists = false;
											}

											stmtA.close();
										}else

										stmtFT.close();
									}

									//Get New
									{
										if(!feedItemExists)
										{
											Log("\tGetting Article        " + Pad(items[i].title,40) + "   " + items[i].link);

											PreparedStatement stmtI = conn.prepareStatement("INSERT INTO " + feedTable + " (Title,Link,ResUrl,Guid,Idx,PubDate,Article,Created,Status) VALUES (?,?,?,?,?,?,?,?,?)");

											HttpRequest hr = null;
											String mainArt = null;
											NodeList nodeList = null;
											boolean usingRssDes = false;

											try
											{
												hr = HttpRequest.Create(items[i].link);
												ExtractMainArticle.ExtractMainArticleNodes(GetPage(hr.getInputStream()),items[i].link,items[i].title);
												nodeList = GetBestMatchingArticle(ExtractMainArticle.GetArticlesByPos(),items[i].content);
												mainArt = nodeList.toHtml();
												mainArt = Truncate(mainArt,32000);
											}catch(Exception ex4)
											{
											    	if(settings.debug)
											    		Log("\t\tFailed To Get Article ",ex4);
											    	else
											    		Log("\t\tFailed To Get Article " + ex4.getMessage());
											}

											if(mainArt!=null && nodeList!=null)
											{
												int desLen = GetDescriptionRealLen(Truncate(items[i].content,32000));
												int artLen = GetArticleRealLen(nodeList.toNodeArray());

												if(desLen > artLen && false)
												{
													mainArt = Truncate(items[i].content,32000);
													usingRssDes = true;
												}
											}

											//Check if article already exists

											String resURL = ((hr==null || hr.getLastURL() == null) ? String.valueOf(System.currentTimeMillis()) : hr.getLastURL());	//In the case we have no ResUrl we need a unique identifier

											boolean articleAlreadyExists = false;
											if( hr!=null && hr.getLastURL() != null )	//If getlastURL==null then resURL=currentTimeMillis which is unique so no need to check for uniquness
											{
												Statement stmtC = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY,ResultSet.CONCUR_UPDATABLE);
												ResultSet resC = stmtC.executeQuery("Select * From Articles Where ResUrl='" + resURL + "'");

												if(resC.next())
												{
													articleAlreadyExists = true;

													if(!Util.StrNullOrEmpty(mainArt) && mainArt.length() > resC.getString("Article").length() * 0.1)
													{
														if(mainArt.equals(resC.getString("Article")))
														{
															Log("\t\tAlready Have This Article.Both Versions Match");
														}else
														{
															Log("\t\tAlready Have This Article.Updating To New Version");

															if(usingRssDes)
																Log("\t\tUsing Rss Description");

															resC.updateString("Article",Truncate(mainArt,32000));
															resC.updateTimestamp("Modified",GetCurrentTimestamp());
														}
													}else
														Log("\t\tAlready Have This Article.Not Updating, New Version Is Invalid");


													resC.updateTimestamp("Checked",GetCurrentTimestamp());
													stmtI.setInt(7,resC.getInt("Id")); //Article
													resC.updateRow();
												}

												stmtC.close();
											}

											stmtI.setString(1,items[i].title);	 		//Title
											stmtI.setString(2,items[i].link);			//Link
											stmtI.setString(3,resURL);					//ResUrl
											stmtI.setString(4,items[i].guid);			//Guid
											stmtI.setInt(5,i);							//Idx
											stmtI.setTimestamp(6,new Timestamp(items[i].date.getTime()));	//PubDate
											stmtI.setTimestamp(8,GetCurrentTimestamp());//Created
											stmtI.setInt(9,GetArticleStatus(mainArt));	//Status

											//If we havn't got the article then insert it
											if(!articleAlreadyExists)
											{
												if(usingRssDes)
													Log("\t\tUsing Rss Description");

												PreparedStatement stmtIA = conn.prepareStatement("INSERT INTO Articles (Article,ResUrl,Modified,Checked) VALUES (?,?,?,?)");
												stmtIA.setString(1,(mainArt==null?"":Truncate(mainArt,32000)));
												stmtIA.setString(2,resURL);
												stmtIA.setTimestamp(3,GetCurrentTimestamp());
												stmtIA.setTimestamp(4,GetCurrentTimestamp());
												stmtIA.executeUpdate();
												stmtIA.close();

												Statement stmtID = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY,ResultSet.CONCUR_READ_ONLY);
												ResultSet resID = stmtID.executeQuery("Select Id From Articles Where ResUrl='" + resURL + "'");
												resID.next();

												stmtI.setInt(7,resID.getInt("Id"));
												stmtID.close();
											}

											aUpdateHasOcurred = true;
											stmtI.executeUpdate();
											stmtI.close();
										}
									}
						    	}catch(Exception ex3)
						    	{
									if(ex3 instanceof SQLDataException && StackToStr(ex3,true).indexOf("A truncation error was encountered trying to shrink VARCHAR") > 0)
						    			Log("\t\t"+ex3.toString());
						    		else
						    			Log("\t\tError Checking Article",ex3);
						    	}
						    }

						    if(Util.StrNullOrEmpty(resF.getString("Title")) && rss.feedData.getTitle()!=null)				//Safe to assume the title wont change, so if the user sets\changes the value it will take priority
						    	resF.updateString("Title",Truncate(rss.feedData.getTitle(),1000));

							if(Util.StrNullOrEmpty(resF.getString("Des")) && rss.feedData.getDescription()!=null)
						    	resF.updateString("Des",Truncate(rss.feedData.getDescription(),1000));


							if(aUpdateHasOcurred && rss.feedData.getPublishedDate()==null)
								resF.updateTimestamp("PubDate",GetCurrentTimestamp());
							else if(rss.feedData.getPublishedDate()!=null)
						    	resF.updateTimestamp("PubDate", new Timestamp(rss.feedData.getPublishedDate().getTime()));

						    String selfURL = resF.getString("SelfLink");
						    //Get SelfURL
						    {
							    if(!Util.StrNullOrEmpty(settings.baseURL))
							    {
									String path = GetFilePath(resF.getString("Path"));
									File file = new File(path);

									if(!file.isAbsolute())
									{
										String base = settings.baseURL.trim();

										if(base.endsWith("/"))
											base = base.substring(0,base.length()-1);

										char[] raw = path.toCharArray();
										for (int i = 0; i<raw.length; i++)
									    	if(raw[i]=='\\')
									    		raw[i] = '/';

									    String end = String.valueOf(raw);

										if(end.startsWith("/"))	//Should never be true
											end = end.substring(1);

										selfURL = base + "/" + end;
									}
							    }
						    }

						    resF.updateString("SelfLink",selfURL);
						    resF.updateTimestamp("Updated",GetCurrentTimestamp());
						}catch(FileNotFoundException exFNOF)
						{
							Log("\tCould Not Find Feed");
						}catch(com.sun.syndication.io.ParsingFeedException ex)
						{
							Log("\tFeed Is Not Of Correct Format");
						}catch(Exception ex1)
						{
							if(StackToStr(ex1,false).indexOf("com.sun.syndication.io") > -1)
								Log("\tFeed Is Not Of Correct Format");
							else
								Log("\tError Checking Feed",ex1);
						}
	        		}

	        		//Delete Entries
	        		{
	        			try
	        			{
		        			Statement stmt = conn.createStatement(ResultSet.TYPE_SCROLL_SENSITIVE,ResultSet.CONCUR_UPDATABLE);
							int countA = stmt.executeUpdate("UPDATE Articles SET DisDate='" + GetCurrentTimestamp() +"' WHERE EXISTS (SELECT ID FROM %FEEDTABLE% WHERE Articles.ID=%FEEDTABLE%.Article AND %FEEDTABLE%.Idx<0)".replaceAll("%FEEDTABLE%",feedTable));
		        			int countB = stmt.executeUpdate("DELETE FROM " + feedTable + " WHERE Idx<0");
				        	stmt.close();

							//if(countA != countB)	//This may not indicate corruption of db, it may occure that the feed item changes its guid, but keeps the same ResUrl
								//Log("\tThe Database Seems Corrupted.An Unequal Amount Of Articles Where Dissasociated As Entires Where Removed. " + countA + " vs " + countB + "\n",System.err);

			        		if(countB > 0 || settings.debug)
			        			Log("\t" + countB + " entrie" + (countB==1?" was":"s were") + " deleted");
	        			}catch(Exception ex6)
	        			{
	        				Log("Error While Deleting Entries",ex6);
	        			}
	        		}

					//Write Feed
	        		{
						try
					    {
					   		String filePath = GetFilePath(resF.getString("Path"));

	    					File file = new File(filePath);

							if(!Util.StrNullOrEmpty(settings.feedsDir) && !file.isAbsolute())
								file = new File(settings.feedsDir,filePath);

					    	boolean writeFeed = false;

					    	if(!writeFeed)
					    		writeFeed = aUpdateHasOcurred;

					    	if(!writeFeed)
					    		writeFeed = !file.exists();

					    	if(!writeFeed)
					    		writeFeed =  ( (System.currentTimeMillis() - resF.getTimestamp("LastWrite").getTime() ) > (1000 * 60 * 60 * 24 * 1) ); //Write out atleast once a day

	        				if(writeFeed)
	        				{
	        					Log("\tWriting Feed");

	    						File parentDir = file.getParentFile();

								boolean dirCreated = true;

	    						if(parentDir!=null && !parentDir.exists())	//If parentDir==null its a relative path, assume workingDir exists
	    							dirCreated = parentDir.mkdirs();

								if(dirCreated)
								{
							    	Statement stmtW = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY,ResultSet.CONCUR_READ_ONLY);
							    	ResultSet resW = stmtW.executeQuery("SELECT Articles.Article,Guid,Link,%FEEDTABLE%.PubDate,Title from Articles,%FEEDTABLE% where Articles.ID=%FEEDTABLE%.Article AND Idx>-1 ORDER BY Idx ASC".replaceAll("%FEEDTABLE%",feedTable));

							        FeedCreator feedOut = new FeedCreator(file.getAbsolutePath());
							        feedOut.WriteHeader(resF.getString("Title"),resF.getString("Des"),CleanURL(resF.getString("Link")),CleanURL(resF.getString("SelfLink")),new java.util.Date(resF.getTimestamp("PubDate").getTime()));

									while(resW.next())
									{
										RssItem item = new RssItem();
										item.content = resW.getString("Article");
										item.date = new java.util.Date(resW.getTimestamp("PubDate").getTime());
										item.guid = resW.getString("Guid");
										item.link = CleanURL(resW.getString("Link"));
										item.title = resW.getString("Title");

										feedOut.AppendItem(item);
									}

									stmtW.close();
									feedOut.WriteFooter();
									feedOut.Close();

									resF.updateTimestamp("LastWrite",GetCurrentTimestamp());
		        				}else
		        				{
		        					Log("Couldn't create parent directory");
		        				}
	        				}
					    }catch (Exception ex1)
					    {
					    	Log("\tError While Writing Feed",ex1);
					    }
	        		}

		        	resF.updateRow();
		        }

		        stmtF.close();
		    }catch (Exception ex)
		    {
		    	Log("A Major Error Has Ocurred",ex);
		    }
	    }

	    //Clean Up
	    {
	    	//Delete Un-needed %FEED% Tables
	    	{
	    		try
	    		{
	    			PreparedStatement stmtQ = conn.prepareStatement("SELECT Id FROM RssFeeds WHERE Id=?",ResultSet.TYPE_FORWARD_ONLY,ResultSet.CONCUR_READ_ONLY);

					int[] tableId = GetFeedTableIDs(conn);

	    			for (int i = 0; i<tableId.length; i++)
				    {
				    	String feedTable = GetFeedTableName(tableId[i]);

				    	try
				    	{
					    	stmtQ.setInt(1,tableId[i]);

					    	ResultSet resQ = stmtQ.executeQuery();

					    	if(!resQ.next())
					    	{
								Statement stmtD = conn.createStatement();
					    		stmtD.executeUpdate("DROP TABLE " + feedTable);
					    		stmtD.close();

				    			Log("Deleted Table " + feedTable);
					    	}

					    	resQ.close();
				    	}catch(Exception ex7)
				    	{
				    		Log("Error While Deleting " + feedTable,ex7);
				    	}
				    }

				    stmtQ.close();
	    		}catch(Exception e)
	    		{
	    			Log("An Error Has Ocurred While Deleting Un-needed Tables",e);
	    		}
	    	}

	    	//Delete Un-needed Articles
	    	{
	    		try
	    		{
		    		Debug("Deleting Articles");

		    		int[] tableIds = GetFeedTableIDs(conn);

					int count = 0;
					if(tableIds.length > 0)
					{
			    		String sqlDel = "DELETE FROM Articles WHERE NOT EXISTS (%INSERT%) AND DisDate<'" + new Timestamp(System.currentTimeMillis() - settings.articleStaleTime) + "'";
						String sqlSel = "SELECT * FROM %FEED% WHERE %FEED%.Idx>=0 AND Articles.ResUrl = %FEED%.ResUrl";

						String insert = "";
						for (int i = 0; i<tableIds.length; i++)
					    	insert += " UNION " + sqlSel.replaceAll("%FEED%",GetFeedTableName(tableIds[i]));

						insert = insert.replaceFirst("UNION","").trim();

						String sql = sqlDel.replaceAll("%INSERT%",insert);

						Statement stmt = conn.createStatement();
						count = stmt.executeUpdate(sql);
						stmt.close();
					}

					if(count > 0 || settings.debug)
					{
		    			Log(count + " article" + (count==1?" was":"s were") + " deleted");
					}
	    		}catch(Exception ex2)
	    		{
	    			Log("Error While Deleting Un-needed Articles",ex2);
	    		}
	    	}
	    }

	    Log("Done");

		//conn.commit();
		conn.close();
    }

    private static String GetFeedTableName(int id)
    {
    	return "FEED" + String.format("%04d",id);
    }

	private static int[] GetFeedTableIDs(Connection conn) throws java.sql.SQLException
	{
		ArrayList<Integer> tablesList = new ArrayList<Integer>();

		Statement stmtT = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY,ResultSet.CONCUR_READ_ONLY);
		ResultSet resT = stmtT.executeQuery("show tables");

		while(resT.next())
		{
			try
			{
				String tableName = resT.getString(1);

				if(tableName.startsWith("FEED"))
					tablesList.add(new Integer(Integer.parseInt((tableName.substring("FEED".length())))));
			}catch(Exception e)
			{
				//Assuming Another Table Has Manually Been Put In
			}
		}

		stmtT.close();

		int[] ids = new int[tablesList.size()];

		for (int i = 0; i<ids.length; i++)
	    	ids[i] = tablesList.get(i).intValue();

		return ids;
	}

	private static int GetDescriptionRealLen(String html) throws Exception
	{
		Parser p = Parser.createParser(html,null);

		Node[] nodes = p.parse(null).toNodeArray();

		return GetArticleRealLen(nodes);
	}

	private static int GetArticleRealLen(Node[] nodes)
	{
		int len = 0;

		for (int i = 0; i<nodes.length; i++)
	    {
	    	String text = nodes[i].toPlainTextString();

	    	if(text!=null)
	    		len+=text.length();
	    }

		return len;
	}

    private static int GetArticleStatus(String article)
    {
    	if(article == null)
    		return ARTICLE_ERROR;
		else if(article.length() < settings.ARTICLE_MIN_LEN)
			return ARTICLE_UPDATE;
		else
			return ARTICLE_NORM;
    }

    private static Timestamp GetCurrentTimestamp()
    {
    	return new Timestamp(System.currentTimeMillis());
    }

    private static boolean CreateTable(Connection conn,String name,String sql) throws Exception
    {
    	if(!Exists(conn,name))
    	{
			Log("Creating Table: " + name);

    		Statement stmt = conn.createStatement();
    		stmt.executeUpdate(sql,Statement.NO_GENERATED_KEYS);
    		stmt.close();
    		return true;
    	}

    	return false;
    }

    private static boolean Exists(Connection conn,String name) throws Exception
    {
		CallableStatement stmt = conn.prepareCall("show tables");
		ResultSet rs = stmt.executeQuery();

		while(rs.next())
		{
			if(rs.getString(1).equals(name))
				return true;
		}

		return false;
    }

    private static String GetFilePath(String filePath) //Converts \ OR / to java.io.File.separatorChar
    {
    	char[] raw = filePath.toCharArray();

    	char ps = java.io.File.separatorChar;
    	for (int i = 0; i<raw.length; i++)
	    {
	    	if(raw[i]=='\\' || raw[i]=='/')
	    		raw[i] = ps;
	    }

	    return String.valueOf(raw);
    }

    private static String CleanURL(String link)
    {
    	try
    	{
    		return URIUtil.encodeQuery(link);
    	}catch(Exception e)
    	{
    		Log("Link Encoding Error",e);
    		return link;
    	}
    }

	//Return what appers to be the main article based on the new information of the feed item
	private static NodeList GetBestMatchingArticle(ArrayList<NodeList> articles,String feedItem)
	{
		if(articles.size() > 0)
		{
			if(articles.size()==1)
				return articles.get(0);

			if(!Util.StrNullOrEmpty(feedItem) && feedItem.length()>15)
			{
				try
				{
					String itemText = RemoveDirtyWords(GetPlainTextString(Parser.createParser(feedItem,null).parse(null))).toLowerCase();

					int maxSearch = Math.min(articles.size(),3);

					int minIdx = -1;
					double minDis = -1;
					for(int i=0;i<maxSearch && minDis!=0;i++)
					{
						String artText = RemoveDirtyWords(GetPlainTextString(articles.get(i))).toLowerCase();

						int minLen = Math.min(itemText.length(),artText.length());

						String itemTextInner = itemText.substring(0,minLen);
						artText = artText.substring(0,minLen);

						double dis = Distance.GetLevenshteinDistance(itemTextInner,artText) / Math.pow(0.75,i+1); //Find the distance between the two strings, '1/0.75^(i+1)' weights in favour of blocks of text appering earlier
						dis = (dis * 100) / (minLen);

						if(dis<minDis || minIdx<0)
						{
							minIdx = i;
							minDis = dis;
						}
					}

					return articles.get(minIdx);
				}catch(Exception e)
				{
					return articles.get(0);
				}
			}else
				return articles.get(0);
		}else
			return new NodeList();
	}

	//Strip out all words which have a char not in the alaphabet
	private static String RemoveDirtyWords(String str)
	{
		String[] split = str.split("\\s+");

		StringBuffer buf = new StringBuffer();

		int count = 0;
		for (int i = 0; i<split.length; i++)
	    {
	    	boolean add = true;

	    	char[] chars = split[i].toCharArray();
	    	for(int j=0;j<chars.length && add;j++)
	    	{
	    		char c = chars[j];
				add = (c>='a' && c<='z') || (c>='A' && c<='Z');
	    	}

			if(add)
				buf.append(split[i]).append(" ");
	    }

	    return buf.toString();
	}

	private static String GetPlainTextString(NodeList nodes)
	{
		StringBuffer buffer = new StringBuffer();

		for(int i=0;i<nodes.size();i++)
		{
			Node node = nodes.elementAt(i);

			if(node instanceof TextNode || node instanceof TagNode)
				buffer.append(node.toPlainTextString());
		}

		return buffer.toString();
	}

    private static String Pad(String str,int len)
    {
    	return String.format("%1$-" + len + "s",str.length() > len ? str.substring(0,len) : str);
    }

	private static String GetPage(String link) throws Exception
	{
		return GetPage( GetStream(link) );
	}

	private static String GetPage(InputStream str) throws Exception
	{
		BufferedReader strIn = new BufferedReader( new InputStreamReader(str,"UTF-8") );

		StringBuilder builder = new StringBuilder();

		int byteRead;
		while ((byteRead = strIn.read()) != -1)
	              	builder.append((char) byteRead);

		strIn.close();

		return builder.toString();
	}

	private static InputStream GetStream(String link) throws Exception
	{
		InputStream strIn = null;

		boolean isFileStream = false, isUrlStream = false;

		if(!isUrlStream)
		{
			try
			{
				strIn = new FileInputStream(link);

				isFileStream = true;
			}catch(Exception e){}
		}

		if(!isFileStream)
		{
			try
			{
				URLConnection urlc = (new URL(link) ).openConnection();
				strIn = urlc.getInputStream();

				isUrlStream = true;
			}catch(Exception e)
			{
			}
		}


		if( !isUrlStream && !isFileStream)
			throw new FileNotFoundException("Page Not Found");
		else
			return strIn;
	}

	private static String GetPath(String parent,String file)
	{
		if(parent == null)
			return file;
		else
			return new File(parent,file).getAbsolutePath();
	}

    private static void Bomb(String mess,Throwable t)
    {
    	Log(mess,t,System.err);
    	Log("Aborting");
    	System.exit(0);
    }

    private static void Bomb(String mess)
    {
    	Log(mess,System.err);
    	Log("Aborting");
    	System.exit(0);
    }

    public static void Debug(String mess)
    {
    	Log(mess,true,System.out);
    }

    public static void Log(String mess,PrintStream str)
    {
    	Log(mess,false,str);
    }

    public static void Log(String mess,Throwable t)
    {
		Log(mess,t,System.err);
    }

    public static void Log(String mess,Throwable t,PrintStream str)
    {
    	Log(mess,t,false,str);
    }

    public static void Log(String mess,Throwable t,boolean debug,PrintStream str)
    {
		Log(mess + "\nException:\n" + StackToStr(t,true),debug,str );
    }

	public static void Log(String mess)
	{
		Log(mess,false,System.out);
	}

	public static void Log(String mess,boolean debug)
	{
		Log(mess,debug,System.out);
	}

	public static void Log(String mess,boolean debug,PrintStream str)
	{
		if(!debug || (debug && settings.debug))
		{
			str.println(mess);
		}
	}

	private static String Truncate(String str,int len)
	{
		if(str.length()<=len)
			return str;
		else
			return str.substring(0,len);
	}

	private static String StackToStr(Throwable t,boolean truncate)
	{
		try
		{
		  	StringWriter sw = new StringWriter();
			PrintWriter pw = new PrintWriter(sw, true);
		 	t.printStackTrace(pw);
			pw.flush();
			sw.flush();

			String stackTrace = sw.toString();

			int idx = stackTrace.indexOf("Caused by");	//Removes the excess trace
			if(idx>-1 && truncate)
				return stackTrace.substring(0,idx).trim();
			else
				return stackTrace;
		}catch(Exception e)
		{
			return "Failed to convert throwable to string.";
		}
	}
}

class Log4jPrintStream extends PrintStream
{
	public Log4jPrintStream(OutputStream strOut)
	{
		super(strOut);
	}

	public void println(String str)
	{
		if(!str.startsWith("log4j:WARN") )
			super.println(str);

	}
}
