Then Came The Cache

[alexander]

So if you run a high-throughput production web servers, then chances are, you have some sort of cache control :) If you serve files, a lot of files, then maybe you have thought about how cool it might be to serve files that are most actively accessed out of cache. If you have, cool, if you are wondering how such a script will work, well, here is an example :)

 

<?php

include_once "memcache_settings.php";

$memcache=new Memcache();

global $memcached_servers;

$servers = (isset($memcached_servers)) ? $memcached_servers : array('127.0.0.1');
$timeout = (isset($timeout)) ? $timeout : 86400;
$expire = (isset($expire)) ? $expire : -1;

foreach($servers as $server) {
list($server, $port) = preg_split("/:/", $server);
!empty($server) || $server='127.0.0.1';
!empty($port) || $port='11211';
if(!$memcache->addServer($server, $port))  { error_log("Could not connect to MemCache server {$server}:{$port}"); }
}

unset($servers);

$file_id = $_GET['get_file'];
$path = $_SERVER['DOCUMENT_ROOT'].addslashes($file_path);

$file_contents=$memcache->get($file_id);
$file_cached=false;
if($file_contents===false) {
$size = @filesize($path);
$handle = @fopen($path, "r");
$file_contents = @fread($handle, $size);
if(!$file_contents)  { error_log("Failed to read file {$path}"); }
fclose($handle);
if(!$memcache->add($file_id, $file_contents, $timeout)) {
	error_log("Failed to cache file {$file_id}");
}
}

if(!$size) { $size=strlen($file_contents); }
header("Content-type: ".mimeType(preg_match("[^\.^\s^\W]+$",$file_path)));
header("Content-Length: {$size}"); // provide file size
header("Expires: {$expire}");
header("Cache-Control: no-store, no-cache, must-revalidate");
header("Cache-Control: post-check=0, pre-check=0", false);
echo $file_contents;

function mimeType($type)
{
$mimeTypes=array(
	'ai' => 'application/postscript', 'bcpio' => 'application/x-bcpio', 'bin' => 'application/octet-stream',
	'ccad' => 'application/clariscad', 'cdf' => 'application/x-netcdf', 'class' => 'application/octet-stream',
	'cpio' => 'application/x-cpio', 'cpt' => 'application/mac-compactpro', 'csh' => 'application/x-csh',
	'csv' => 'application/csv', 'dcr' => 'application/x-director', 'dir' => 'application/x-director',
	'dms' => 'application/octet-stream', 'doc' => 'application/msword', 'drw' => 'application/drafting',
	'dvi' => 'application/x-dvi', 'dwg' => 'application/acad', 'dxf' => 'application/dxf',
	'dxr' => 'application/x-director', 'eot' => 'application/vnd.ms-fontobject', 'eps' => 'application/postscript',
	'exe' => 'application/octet-stream', 'ez' => 'application/andrew-inset',
	'flv' => 'video/x-flv', 'gtar' => 'application/x-gtar', 'gz' => 'application/x-gzip',
	'bz2' => 'application/x-bzip', '7z' => 'application/x-7z-compressed', 'hdf' => 'application/x-hdf',
	'hqx' => 'application/mac-binhex40', 'ico' => 'image/vnd.microsoft.icon', 'ips' => 'application/x-ipscript',
	'ipx' => 'application/x-ipix', 'js' => 'application/x-javascript', 'latex' => 'application/x-latex',
	'lha' => 'application/octet-stream', 'lsp' => 'application/x-lisp', 'lzh' => 'application/octet-stream',
	'man' => 'application/x-troff-man', 'me' => 'application/x-troff-me', 'mif' => 'application/vnd.mif',
	'ms' => 'application/x-troff-ms', 'nc' => 'application/x-netcdf', 'oda' => 'application/oda',
	'otf' => 'font/otf', 'pdf' => 'application/pdf',
	'pgn' => 'application/x-chess-pgn', 'pot' => 'application/mspowerpoint', 'pps' => 'application/mspowerpoint',
	'ppt' => 'application/mspowerpoint', 'ppz' => 'application/mspowerpoint', 'pre' => 'application/x-freelance',
	'prt' => 'application/pro_eng', 'ps' => 'application/postscript', 'roff' => 'application/x-troff',
	'scm' => 'application/x-lotusscreencam', 'set' => 'application/set', 'sh' => 'application/x-sh',
	'shar' => 'application/x-shar', 'sit' => 'application/x-stuffit', 'skd' => 'application/x-koan',
	'skm' => 'application/x-koan', 'skp' => 'application/x-koan', 'skt' => 'application/x-koan',
	'smi' => 'application/smil', 'smil' => 'application/smil', 'sol' => 'application/solids',
	'spl' => 'application/x-futuresplash', 'src' => 'application/x-wais-source', 'step' => 'application/STEP',
	'stl' => 'application/SLA', 'stp' => 'application/STEP', 'sv4cpio' => 'application/x-sv4cpio',
	'sv4crc' => 'application/x-sv4crc', 'svg' => 'image/svg+xml', 'svgz' => 'image/svg+xml',
	'swf' => 'application/x-shockwave-flash', 't' => 'application/x-troff',
	'tar' => 'application/x-tar', 'tcl' => 'application/x-tcl', 'tex' => 'application/x-tex',
	'texi' => 'application/x-texinfo', 'texinfo' => 'application/x-texinfo', 'tr' => 'application/x-troff',
	'tsp' => 'application/dsptype', 'ttf' => 'font/ttf',
	'unv' => 'application/i-deas', 'ustar' => 'application/x-ustar',
	'vcd' => 'application/x-cdlink', 'vda' => 'application/vda', 'xlc' => 'application/vnd.ms-excel',
	'xll' => 'application/vnd.ms-excel', 'xlm' => 'application/vnd.ms-excel', 'xls' => 'application/vnd.ms-excel',
	'xlw' => 'application/vnd.ms-excel', 'zip' => 'application/zip', 'aif' => 'audio/x-aiff', 'aifc' => 'audio/x-aiff',
	'aiff' => 'audio/x-aiff', 'au' => 'audio/basic', 'kar' => 'audio/midi', 'mid' => 'audio/midi',
	'midi' => 'audio/midi', 'mp2' => 'audio/mpeg', 'mp3' => 'audio/mpeg', 'mpga' => 'audio/mpeg',
	'ra' => 'audio/x-realaudio', 'ram' => 'audio/x-pn-realaudio', 'rm' => 'audio/x-pn-realaudio',
	'rpm' => 'audio/x-pn-realaudio-plugin', 'snd' => 'audio/basic', 'tsi' => 'audio/TSP-audio', 'wav' => 'audio/x-wav',
	'asc' => 'text/plain', 'c' => 'text/plain', 'cc' => 'text/plain', 'css' => 'text/css', 'etx' => 'text/x-setext',
	'f' => 'text/plain', 'f90' => 'text/plain', 'h' => 'text/plain', 'hh' => 'text/plain', 'htm' => 'text/html',
	'html' => 'text/html', 'm' => 'text/plain', 'rtf' => 'text/rtf', 'rtx' => 'text/richtext', 'sgm' => 'text/sgml',
	'sgml' => 'text/sgml', 'tsv' => 'text/tab-separated-values', 'tpl' => 'text/template', 'txt' => 'text/plain',
	'xml' => 'text/xml', 'avi' => 'video/x-msvideo', 'fli' => 'video/x-fli', 'mov' => 'video/quicktime',
	'movie' => 'video/x-sgi-movie', 'mpe' => 'video/mpeg', 'mpeg' => 'video/mpeg', 'mpg' => 'video/mpeg',
	'qt' => 'video/quicktime', 'viv' => 'video/vnd.vivo', 'vivo' => 'video/vnd.vivo', 'gif' => 'image/gif',
	'ief' => 'image/ief', 'jpe' => 'image/jpeg', 'jpeg' => 'image/jpeg', 'jpg' => 'image/jpeg',
	'pbm' => 'image/x-portable-bitmap', 'pgm' => 'image/x-portable-graymap', 'png' => 'image/png',
	'pnm' => 'image/x-portable-anymap', 'ppm' => 'image/x-portable-pixmap', 'ras' => 'image/cmu-raster',
	'rgb' => 'image/x-rgb', 'tif' => 'image/tiff', 'tiff' => 'image/tiff', 'xbm' => 'image/x-xbitmap',
	'xpm' => 'image/x-xpixmap', 'xwd' => 'image/x-xwindowdump', 'ice' => 'x-conference/x-cooltalk',
	'iges' => 'model/iges', 'igs' => 'model/iges', 'mesh' => 'model/mesh', 'msh' => 'model/mesh',
	'silo' => 'model/mesh', 'vrml' => 'model/vrml', 'wrl' => 'model/vrml',
	'mime' => 'www/mime', 'pdb' => 'chemical/x-pdb', 'xyz' => 'chemical/x-pdb','css' => 'text/css','js'=>'text/javascript');
	return $mimeTypes[$type];
}

 

Try it, throw some nginx config around it, and request that file from your server a hundred thousand times in a minute... see if it even flinches :)

 

Alternately to speed this up, you could write this in a faster language, like C, and you could use back end like a Heystack or something, if you are THAT busy :) But for now this is my post and i am sticking to it.