From e0b5b720c968cd95e65c9a5e80b40cab38391b24 Mon Sep 17 00:00:00 2001 From: Dominique Fournier Date: Thu, 25 Feb 2016 07:53:07 +0000 Subject: [PATCH] Add DBJSON support. DBJSON is a NoSQL database, writing the data in one file. There is no optimizations, so it is not quick, but it works on all the PHP sites git-svn-id: https://svn.fournier38.fr/svn/ProgSVN/trunk@2561 bf3deb0d-5f1a-0410-827f-c0cc1f45334c --- Tests/dbjsonTest.php | 351 ++++++++++++++++++++++++++++++++++++++ dbjson.php | 398 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 749 insertions(+) create mode 100644 Tests/dbjsonTest.php create mode 100644 dbjson.php diff --git a/Tests/dbjsonTest.php b/Tests/dbjsonTest.php new file mode 100644 index 0000000..8853b97 --- /dev/null +++ b/Tests/dbjsonTest.php @@ -0,0 +1,351 @@ +insertOne ("collection", + array ("key1"=>"val1", "key2"=>"val2")); + $this->assertSame ($res, 1); + } + + public function test_insertOne2 () + { + // Document #1 + $dbjson = new dbjson ("dbjson://".dbfile); + $res = $dbjson->insertOne ("collection", + array ("key1"=>"val1", "key2"=>"val2")); + $this->assertSame ($res, 1); + } + + public function test_insertMany1 () + { + // Error : Invalid array provided (not array of array) + $this->setExpectedException ("Exception"); + $dbjson = new dbjson ("dbjson://".dbfile); + $res = $dbjson->insertMany ("collection", + array ("key1"=>"val1", "key2"=>"val2")); + } + + public function test_insertMany2 () + { + // Document #2 and #3 + $dbjson = new dbjson ("dbjson://".dbfile); + $res = $dbjson->insertMany ("collection", + array (array ("key1"=>"val3", "key2"=>"val2"), + array ("key1"=>"val3", "key2"=>"val4"))); + $this->assertSame ($res, 2); + } + + public function test_filter1 () + { + // Return all the keys (filter = array ()) + $dbjson = new dbjson ("dbjson://".dbfile); + $res = $dbjson->filter ("collection", array ()); + $this->assertSame ($res, array (0,1,2,3)); + } + public function test_filter2 () + { + // Return the keys where filter = array ("key2"=>"val2")) + $dbjson = new dbjson ("dbjson://".dbfile); + $res = $dbjson->filter ("collection", array ("key2"=>"val2")); + $this->assertSame ($res, array (0,1,2)); + } + public function test_filter3 () + { + // Return the keys where filter = array ("key1"=>"val3","key2"=>"val2")) + $dbjson = new dbjson ("dbjson://".dbfile); + $res = $dbjson->filter ("collection", array ("key1"=>"val3", + "key2"=>"val2")); + $this->assertSame ($res, array (2)); + } + public function test_filter4 () + { + // Return the keys where filter = array ("key1"=>array ("val3", "=="), + // "key2"=>array ("val2", "==")) + $dbjson = new dbjson ("dbjson://".dbfile); + $res = $dbjson->filter ("collection",array ("key1"=>array ("val3", "=="), + "key2"=>array ("val2", "=="))); + $this->assertSame ($res, array (2)); + } + public function test_filter5 () + { + // Return the keys where filter = array ("key1"=>array ("val3", "<="), + // "key2"=>array ("val2", "==")) + $dbjson = new dbjson ("dbjson://".dbfile); + $res = $dbjson->filter ("collection", array ("key1"=>array ("val3", "<="), + "key2"=>array ("val2", "=="))); + $this->assertSame ($res, array (0,1,2)); + } + public function test_filter6 () + { + // Return the keys where filter = array ("key1"=>array ("val3", "<="), + // "key2"=>array ("val2", ">")) + $dbjson = new dbjson ("dbjson://".dbfile); + $res = $dbjson->filter ("collection", array ("key1"=>array ("val3", "<="), + "key2"=>array ("val2", ">"))); + $this->assertSame ($res, array (3)); + } + + public function test_find1 () + { + $dbjson = new dbjson ("dbjson://".dbfile); + $res = $dbjson->find ("collection", array ("key1"=>array ("val3", "<="), + "key2"=>array ("val2", ">"))); + // ["_id"] is random : skip the test + unset ($res[3]["_id"]); + $this->assertSame ($res, array (3=>array ("key1"=>"val3", + "key2"=>"val4"))); + } + public function test_find2 () + { + $dbjson = new dbjson ("dbjson://".dbfile); + $res = $dbjson->find ("collection", array ("key1"=>array ("val3", "<="), + "key2"=>array ("val2", ">")), + "*"); + // ["_id"] is random : skip the test + unset ($res[3]["_id"]); + $this->assertSame ($res, array (3=>array ("key1"=>"val3", + "key2"=>"val4"))); + } + public function test_find3 () + { + // Exception : fields not an array + $this->setExpectedException ("Exception"); + $dbjson = new dbjson ("dbjson://".dbfile); + $res = $dbjson->find ("collection", array ("key1"=>array ("val3", "<="), + "key2"=>array ("val2", ">")), + "key1"); + } + public function test_find4 () + { + $dbjson = new dbjson ("dbjson://".dbfile); + $res = $dbjson->find ("collection", array ("key1"=>array ("val3", "<="), + "key2"=>array ("val2", ">")), + array ("key1")); + // ["_id"] is random : skip the test + unset ($res[3]["_id"]); + $this->assertSame ($res, array (3=>array ("key1"=>"val3"))); + } + public function test_find5 () + { + $dbjson = new dbjson ("dbjson://".dbfile); + $res = $dbjson->find ("collection", array ("key1"=>array ("val3", "<="), + "key2"=>array ("val2", ">")), + array ("key1", "key2")); + // ["_id"] is random : skip the test + unset ($res[3]["_id"]); + $this->assertSame ($res, array (3=>array ("key1"=>"val3", + "key2"=>"val4"))); + } + public function test_find6 () + { + $dbjson = new dbjson ("dbjson://".dbfile); + $res = $dbjson->find ("collection", array ("key1"=>array ("val3", "<="), + "key2"=>array ("val2", "==")), + array ("key1", "key2")); + // ["_id"] is random : skip the test + unset ($res[0]["_id"]); + unset ($res[1]["_id"]); + unset ($res[2]["_id"]); + $this->assertSame ($res, array (0=>array ("key1"=>"val1", + "key2"=>"val2"), + 1=>array ("key1"=>"val1", + "key2"=>"val2"), + 2=>array ("key1"=>"val3", + "key2"=>"val2"))); + } + public function test_find7 () + { + $dbjson = new dbjson ("dbjson://".dbfile); + $res = $dbjson->find ("collection", array ("key1"=>array ("val3", "<="), + "key2"=>array ("val2", "==")), + array ("key2")); + // ["_id"] is random : skip the test + unset ($res[0]["_id"]); + unset ($res[1]["_id"]); + unset ($res[2]["_id"]); + $this->assertSame ($res, array (0=>array ("key2"=>"val2"), + 1=>array ("key2"=>"val2"), + 2=>array ("key2"=>"val2"))); + } + public function test_find8 () + { + $dbjson = new dbjson ("dbjson://".dbfile); + $res = $dbjson->find ("collection", array ("key1"=>array ("val3", "<="), + "key2"=>array ("val2", "==")), + array ("key2"), 1); + // ["_id"] is random : skip the test + unset ($res[0]["_id"]); + $this->assertSame ($res, array (0=>array ("key2"=>"val2"))); + } + + public function test_deleteOne1 () + { + $dbjson = new dbjson ("dbjson://".dbfile); + $res = $dbjson->deleteOne ("collection", + array ("key1"=>array ("val3", "<="), + "key2"=>array ("val2", "=="))); + $this->assertSame ($res, 1); + } + public function test_find9 () + { + $dbjson = new dbjson ("dbjson://".dbfile); + $res = $dbjson->find ("collection", array ("key1"=>array ("val3", "<="), + "key2"=>array ("val2", "==")), + array ("key2")); + // ["_id"] is random : skip the test + unset ($res[1]["_id"]); + unset ($res[2]["_id"]); + $this->assertSame ($res, array (1=>array ("key2"=>"val2"), + 2=>array ("key2"=>"val2"))); + } + + public function test_deleteMany1 () + { + $dbjson = new dbjson ("dbjson://".dbfile); + $res = $dbjson->deleteMany ("collection", + array ("key1"=>array ("val3", "<="), + "key2"=>array ("val2", "=="))); + $this->assertSame ($res, 2); + } + public function test_find10 () + { + $dbjson = new dbjson ("dbjson://".dbfile); + $res = $dbjson->find ("collection", array ("key1"=>array ("val3", "<="), + "key2"=>array ("val2", "==")), + array ("key2")); + // ["_id"] is random : skip the test + unset ($res[1]["_id"]); + unset ($res[2]["_id"]); + $this->assertSame ($res, array ()); + } + + public function test_find11 () + { + $dbjson = new dbjson ("dbjson://".dbfile); + $res = $dbjson->find ("collection"); + // ["_id"] is random : skip the test + unset ($res[3]["_id"]); + $this->assertSame ($res, array (3=>array ("key1"=>"val3", "key2"=>"val4"))); + } + + public function test_replace1 () + { + $dbjson = new dbjson ("dbjson://".dbfile); + $res = $dbjson->replace ("collection", array (), array ("key2"=>"val5")); + $this->assertSame ($res, 1); + } + public function test_find12 () + { + $dbjson = new dbjson ("dbjson://".dbfile); + $res = $dbjson->find ("collection"); + // ["_id"] is random : skip the test + unset ($res[3]["_id"]); + $this->assertSame ($res, array (3=>array ("key2"=>"val5"))); + } + + public function test_update1 () + { + $dbjson = new dbjson ("dbjson://".dbfile); + $res = $dbjson->update ("collection", array (), array ("key2"=>"val6", + "key5"=>"val5")); + $this->assertSame ($res, 1); + } + public function test_find13 () + { + $dbjson = new dbjson ("dbjson://".dbfile); + $res = $dbjson->find ("collection"); + // ["_id"] is random : skip the test + unset ($res[3]["_id"]); + $this->assertSame ($res, array (3=>array ("key2"=>"val6", + "key5"=>"val5"))); + } + public function test_insertOne3 () + { + // Document #4 + $dbjson = new dbjson ("dbjson://".dbfile); + $res = $dbjson->insertOne ("collection", + array ("key1"=>"val1", "key2"=>"val2")); + $this->assertSame ($res, 1); + } + public function test_find14 () + { + $dbjson = new dbjson ("dbjson://".dbfile); + $res = $dbjson->find ("collection"); + // ["_id"] is random : skip the test + unset ($res[3]["_id"]); + unset ($res[4]["_id"]); + $this->assertSame ($res, array (3=>array ("key2"=>"val6", + "key5"=>"val5"), + 4=>array ("key1"=>"val1", + "key2"=>"val2"))); + } + + public function test_update2 () + { + $dbjson = new dbjson ("dbjson://".dbfile); + $res = $dbjson->update ("collection", array (), array ("key2"=>"val7", + "key5"=>"val8")); + $this->assertSame ($res, 2); + } + public function test_find15 () + { + $dbjson = new dbjson ("dbjson://".dbfile); + $res = $dbjson->find ("collection"); + // ["_id"] is random : skip the test + unset ($res[3]["_id"]); + unset ($res[4]["_id"]); + $this->assertSame ($res, array (3=>array ("key2"=>"val7", + "key5"=>"val8"), + 4=>array ("key1"=>"val1", + "key2"=>"val7", + "key5"=>"val8"))); + } + + public function test_update3 () + { + $dbjson = new dbjson ("dbjson://".dbfile); + $res = $dbjson->update ("collection", array (), + array ("key2"=>"val9", + "key5"=>"val7", + "_unset"=>array ("key2"))); + $this->assertSame ($res, 2); + } + public function test_find16 () + { + $dbjson = new dbjson ("dbjson://".dbfile); + $res = $dbjson->find ("collection"); + // ["_id"] is random : skip the test + unset ($res[3]["_id"]); + unset ($res[4]["_id"]); + $this->assertSame ($res, array (3=>array ("key5"=>"val7"), + 4=>array ("key1"=>"val1", + "key5"=>"val7"))); + } + + // Concurrency tests + public function test_concurrency1 () + { + $dbjson1 = new dbjson ("dbjson://".dbfile); + $dbjson2 = new dbjson ("dbjson://".dbfile); + $dbjson1->insertOne ("collection", + array ("key1"=>"val1", "key2"=>"val2")); + $res = $dbjson2->find ("collection"); + // ["_id"] is random : skip the test + unset ($res[3]["_id"]); + unset ($res[4]["_id"]); + unset ($res[5]["_id"]); + $this->assertSame ($res, array (3=>array ("key5"=>"val7"), + 4=>array ("key1"=>"val1", + "key5"=>"val7"), + 5=>array ("key1"=>"val1", + "key2"=>"val2"))); + } +} diff --git a/dbjson.php b/dbjson.php new file mode 100644 index 0000000..731c790 --- /dev/null +++ b/dbjson.php @@ -0,0 +1,398 @@ +"val") <== Look for the key equal val + * array ("key=>array ("val", "<=")) <== Look for the key lighter or equal + * than val + * array () <== Look for all the documents (no + * filter) + * array ("key"=>"val", "key2"=>"val2") <== Look for two parameters + * - A document is an array containing the fields and the values to store + * array ("key"=>"val) + * + * - The field named _id is the document key + */ + +/** DBJSON : a NoSQL DB in JSON */ +class dbjson +{ + private $dsn = ""; + private $dbfile = ""; + private $dbfileLock = ""; + private $lastInsertId = 0; + // The database content + private $db; + + /** The constructor */ + public function __construct ($dsn) + { + if (! function_exists ("openssl_random_pseudo_bytes")) + throw new \Exception ("Function openssl_random_pseudo_bytes missing", + 500); + $pos = strpos ($dsn, "://"); + if ($pos === false) + throw new \Exception (_("No DSN provided to dbjson"), 500); + if (substr ($dsn, 0, $pos) !== "dbjson") + throw new \Exception (_("Invalid database type provided in dbjson"), 500); + $this->dbfile = substr ($dsn, $pos+3); + $directory = dirname ($this->dbfile); + if (! file_exists ($directory)) + @mkdir ($directory, 0777, true); + if (! file_exists ($directory)) + throw new \Exception (sprintf (_("Directory '%s' doesn't exists"), + $directory), 500); + if (! file_exists ($this->dbfile)) + { + if (! is_readable ($directory)) + throw new \Exception (sprintf ( + _("Directory '%s' not writeable and dbfile '%s' not exists"), + $directory, $this->dbfile), 500); + touch ($this->dbfile); + } + if (! is_readable ($this->dbfile)) + throw new \Exception(sprintf (_("File '%s' not readable"), $this->dbfile), + 500); + if (! is_writeable ($this->dbfile)) + throw new \Exception(sprintf (_("File '%s' not readable"), $this->dbfile), + 500); + $this->dsn = $dsn; + $this->dbfile = $this->dbfile; + } + + /** Store one document in database + * @param $collection string The collection name + * @param $document array() + * @return integer return The number of document inserted in the database + */ + public function insertOne ($collection, $document) + { + $this->lockEX (); + $this->db = $this->readDB (); + $this->db[$collection]["content"][] = array_merge (array ( + "_id"=>$this->uniqueKey ()), + $document); + $this->writeDB ($this->db); + $this->lockUN (); + return 1; + } + + /** Store multiple documents in database + * @param $collection string The collection name + * @param $documents array(array ()) + * @return integer The number of documents inserted in the database + */ + public function insertMany ($collection, $documents) + { + foreach ($documents as $document) + if (! is_array ($document)) + throw new \Exception ("insertMany need an array of array", 406); + $this->lockEX (); + $this->db = $this->readDB (); + foreach ($documents as $document) + { + $this->db[$collection]["content"][] = array_merge (array ( + "_id"=>$this->uniqueKey ()), + $document); + } + $this->writeDB ($this->db); + $this->db = null; + $this->lockUN (); + return count ($documents); + } + + /** Look at the documents matching $filter (all by default). + * Then return only the $fields (all by default). + * The field _id is always returned + * Return $limit maximum documents (no limit by default) + * @param $collection string The collection name + * @param $filter array The filter to apply to found the documents + * @param $fields array|string The fields to display (* for all, empty array + * for none) + * @param $limit integer The number of documents to display + * @return array The documents matching the parameters + */ + public function find ($collection, $filter = array (), $fields = "*", + $limit = null) + { + $this->lockSH (); + $this->db = $this->readDB (); + // Get the keys of the documents based on the filter + $keys = $this->filter ($collection, $filter); + $res = array (); + foreach ($keys as $key) + { + // Limit the fields + $tmp = array (); + if ($fields === "*") + $tmp = $this->db[$collection]["content"][$key]; + elseif (is_array ($fields)) + { + if (! in_array ("_id", $fields)) + array_unshift ($fields, "_id"); + foreach ($fields as $field) + { + if (array_key_exists ($field, + $this->db[$collection]["content"][$key])) + $tmp[$field] = $this->db[$collection]["content"][$key][$field]; + } + } + else + throw new \Exception ("Invalid field list provided", 500); + $res[$key] = $tmp; + // Limit the number of results + if ($limit !== null && count ($res) >= $limit) + break; + } + $this->lockUN (); + $this->db = null; + return $res; + } + + /** Update some existing documents. Do not change the _id keys + * @param $collection string The collection name + * @param $filter array The filter to apply to found the documents + * @param $document array The data to update + * @return integer The number of modified documents + * To unset a field, add in the document array a "_unset"=>array("field)" + */ + public function update ($collection, $filter, $document) + { + $this->lockEX (); + $this->db = $this->readDB (); + // Get the keys of the documents based on the filter + $keys = $this->filter ($collection, $filter); + $unset = array (); + if (array_key_exists ("_unset", $document)) + { + $unset = $document["_unset"]; + unset ($document["_unset"]); + } + foreach ($keys as $key) + { + // Merge the new document with the old + $tmp = $this->db[$collection]["content"][$key]; + $tmp = array_merge ($tmp, $document); + $this->db[$collection]["content"][$key] = $tmp; + // Remove the needed unset fields + foreach ($unset as $field) + if (array_key_exists ($field, + $this->db[$collection]["content"][$key])) + unset ($this->db[$collection]["content"][$key][$field]); + } + $this->writeDB ($this->db); + $this->lockUN (); + $this->db = null; + return count ($keys); + } + + /** Replace some existing documents. Do not change the _id keys + * @param $collection string The collection name + * @param $filter array The filter to apply to found the documents + * @param $document array The data to update + * @return integer The number of modified documents + */ + public function replace ($collection, $filter, $document) + { + $this->lockEX (); + $this->db = $this->readDB (); + // Get the keys of the documents based on the filter + $keys = $this->filter ($collection, $filter); + foreach ($keys as $key) + { + $tmp = $this->db[$collection]["content"][$key]; + $replace = array (); + $replace["_id"] = $tmp["_id"]; + $replace = array_merge ($replace, $document); + $this->db[$collection]["content"][$key] = $replace; + } + $this->writeDB ($this->db); + $this->lockUN (); + $this->db = null; + return count ($keys); + } + + /** Delete the first document matching the filter + * @param $collection string The collection name + * @param $filter array The filter to found the documents + * @return integer The number of deleted documents + */ + public function deleteOne ($collection, $filter) + { + $this->lockEX (); + $this->db = $this->readDB (); + // Get the keys of the documents based on the filter + $keys = $this->filter ($collection, $filter); + if (count ($keys) === 0) + return 0; + reset ($keys); + $key = key ($keys); + unset ($this->db[$collection]["content"][$key]); + $this->writeDB ($this->db); + $this->lockUN (); + $this->db = null; + return 1; + + } + + /** Delete all the documents matching the filter + * @param $collection string The collection name + * @param $filter array The filter to apply to found the documents + * @return integer The number of deleted documents + */ + public function deleteMany ($collection, $filter) + { + $this->lockEX (); + $this->db = $this->readDB (); + // Get the keys of the documents based on the filter + $keys = $this->filter ($collection, $filter); + foreach ($keys as $key) + unset ($this->db[$collection]["content"][$key]); + $this->writeDB ($this->db); + $this->lockUN (); + $this->db = null; + return count ($keys); + } + + /** Look for the keys corresponding to the filter in the collection + * Don't manage the locks ! + * @param $collection string The collection name + * @param $filter array The filter to apply to found the documents + * - A filter is an array containing the fields and the values to found + * array () <== Look for all the documents (no + * filter) + * array ("key"=>"val") <== Look for the key equal val + * array ("key=>array ("val", "<=")) <== Look for the key lighter or + * equal than val + * array ("key"=>"val", "key2"=>"val2") <== Look for two parameters + * array ("key"=>array ("val", "=="), + * "key2"=>array ("val2", "==")) <== Look for two complex parameters + * Here is the comparison types available : ==, + * @return array the keys matching the filter + */ + public function filter ($collection, $filter) + { + if ($this->db === null) + $this->db = $this->readDB (); + $keys = array (); + foreach ($this->db[$collection]["content"] as $key=>$document) + { + if ($filter === array ()) + { + $keys[] = $key; + continue; + } + $matchFilter = false; + foreach ($filter as $fkey=>$fvals) + { + if (is_array ($fvals)) + { + // $fvals = array ("key=>array ("val", "<=")) + if (array_key_exists ($fkey, $document)) + { + if ($fvals[1] !== "==" && + $fvals[1] !== "<=" && + $fvals[1] !== ">=" && + $fvals[1] !== "<" && + $fvals[1] !== ">" && + $fvals[1] !== "exists" && + $fvals[1] !== "not exists") + throw new \Exception ("Invalid filter operator provided", 500); + if ($fvals[1] === "==" && $document[$fkey] === $fvals[0]) + $matchFilter = true; + elseif ($fvals[1] === "<=" && $document[$fkey] <= $fvals[0]) + $matchFilter = true; + elseif ($fvals[1] === ">=" && $document[$fkey] >= $fvals[0]) + $matchFilter = true; + elseif ($fvals[1] === "<" && $document[$fkey] < $fvals[0]) + $matchFilter = true; + elseif ($fvals[1] === ">" && $document[$fkey] > $fvals[0]) + $matchFilter = true; + elseif (strtolower ($fvals[1]) === "exists" && + array_key_exists ($fkey, $document)) + $matchFilter = true; + elseif (strtolower ($fvals[1]) === "not exists" && + ! array_key_exists ($fkey, $document)) + $matchFilter = true; + else + { + $matchFilter = false; + break; + } + } + } + else + { + // $fvals = array ("key"=>"val") + if (array_key_exists ($fkey, $document) && + $document[$fkey] === $fvals) + $matchFilter = true; + else + { + $matchFilter = false; + break; + } + } + } + if ($matchFilter === true) + $keys[] = $key; + } + return $keys; + } + + /** Generate a unique key + * @return string the Unique key generated + */ + private function uniqueKey () + { + $data = openssl_random_pseudo_bytes(16); + $data[6] = chr(ord($data[6]) & 0x0f | 0x40); // set version to 0010 + $data[8] = chr(ord($data[8]) & 0x3f | 0x80); // set bits 6-7 to 10 + return vsprintf('%s%s-%s-%s-%s-%s%s%s', str_split(bin2hex($data), 4)); + } + + /** Exclusive lock the database file */ + private function lockEX () + { + $this->dbfileLock = fopen ($this->dbfile, "rt"); + if (flock ($this->dbfileLock, LOCK_EX) === false) + throw new \Exception ("Can't get exclusive lock on dbfile", 500); + } + + /** Shared lock the database file */ + private function lockSH () + { + $this->dbfileLock = fopen ($this->dbfile, "rt"); + if (flock ($this->dbfileLock, LOCK_SH) === false) + throw new \Exception ("Can't get shared lock on dbfile", 500); + } + + /** Unlock the database file */ + private function lockUN () + { + if ($this->dbfileLock !== null) + { + flock ($this->dbfileLock, LOCK_UN); + fclose ($this->dbfileLock); + $this->dbfileLock = null; + } + } + + /** Read the dbfile and return an array containing the data. This function + * don't do locks ! + * @return array The database content from the dbfile + */ + private function readDB () + { + return json_decode (file_get_contents ($this->dbfile), true); + } + + /** Write the dbfile with the provided data. This function don't do locks ! + * @param $data array The database to store + * @return bool True if the recording is OK, false if there is a problem + */ + private function writeDB ($data) + { + return !! file_put_contents ($this->dbfile, json_encode ($data)); + } +}