Wednesday, March 26, 2014

Automatically upload a screenshot from the clipboard on paste using Javascript in Google Chrome, Internet Explorer 11 and Firefox

I was working on a new documentation site and had to make a lot of screenshots. I thought it would be useful to be able to hit print screen and then paste the image directly into Group-Office using the CTRL+V shortcut. Luckily with HTML5 you can do that!

With Google Chrome and Internet Explorer 11 you can use the Clipboard API. I created a "Paster" class that you can pass a DOM object and a callback function. So when you paste an image into that object the file is uploaded to a PHP script and then the callback function is called with the response.

In this working example it will call the PHP page itself and then put the response in the textarea. It should work on any PHP enabled web server.

You can add an onpaste event to any DOM element in Google Chrome. Then the "Paster" loops through all clipboard items and checks if there's an image:


Paster.prototype.handlePaste = function(paster, e) {

 //don't do this twice
 if (paster.processing) {
  return;
 }

 //loop through all clipBoardData items and upload it if it's a file.
 for (var i = 0; i < e.clipboardData.items.length; i++) {
  var item = e.clipboardData.items[i];
  if (item.kind === "file") {

   paster.processing = true;
   e.preventDefault();
   paster.uploadFile(paster, item.getAsFile());
  }
 }
};

Then it uploads the file using an "XMLHttpRequest" call. When the upload is finished the callback function is called.

Paster.prototype.uploadFile = function(paster, file) {

 var xhr = new XMLHttpRequest();

 //progress logging
 xhr.upload.onprogress = function(e) {
  var percentComplete = (e.loaded / e.total) * 100;

  console.log(percentComplete);
 };

 //called when finished
 xhr.onload = function() {
  if (xhr.status === 200) {
   alert("Sucess! Upload completed. PHP response will be put in the textarea.");
  } else {
   alert("Error! Upload failed");
  }

  paster.processing = false;
 };

 //error handling
 xhr.onerror = function() {
  alert("Error! Upload failed. Can not connect to server.");
 };

 //trigger a callback when it's successful
 xhr.onreadystatechange = function()
 {
  if (xhr.readyState === 4 && xhr.status === 200)
  {
   if (paster.callback) {
    paster.callback.call(paster.scope || paster, paster, xhr);
   }
  }
 };


 //prompt for the filename
 var filename = prompt("Please enter the file name", "Pasted image");

 if (filename) {

  //upload the file
  xhr.open("POST", "", {
   filename: filename,
   filetype: file.type
  });
  var formData = new FormData();
  formData.append("pastedFile", file);
  xhr.send(formData);
 }
}

Firefox

So far it was pretty simple! But then I started testing this in Firefox. Unfortunately it didn't work because Firefox doesn't support files in the clipboard. I came up with a work around though.

You can paste images in a content editable div in Firefox. The blob data is then available in the image source tag: <img src="data:image/png;base64,DATA" />. So in my workaround I create a hidden content editable div. When ever the textarea blurs, it will focus this hidden element. So when you click out of the textarea and then hit CTRL+V, the image is pasted in the hidden div. The paster then grabs that image source and creates a file blob object. This file can be uploaded just like in Google Chrome. Not as user friendly because you have to paste outside the textarea but it works:

Paster.prototype.init=function() {

 var paster = this;

 if (window.Clipboard) {
  //IE11, Chrome, Safari
  this.pasteEl.onpaste=function(e){
   paster.handlePaste(paster, e);
  };
 } else
 {
  //On Firefox use the contenteditable div hack
  this.canvas = document.createElement('canvas');
  this.pasteCatcher = document.createElement("div");
  this.pasteCatcher.setAttribute("id", "paste_ff");
  this.pasteCatcher.setAttribute("contenteditable", "");
  this.pasteCatcher.style.cssText = 'opacity:0;position:fixed;top:0px;left:0px;';
  this.pasteCatcher.style.marginLeft = "-20px";
  document.body.appendChild(this.pasteCatcher);


  this.pasteEl.onblur=function(e) {
   paster.pasteCatcher.focus();
  };

  this.pasteCatcher.onpaste=function(e) {

   paster.findImageEl(paster);
  };
 }
};

 
Paster.prototype.dataURItoBlob=function(dataURI, callback) {
// convert base64 to raw binary data held in a string
// doesn't handle URLEncoded DataURIs - see SO answer #6850276 for code that does this

 console.log(dataURI);

 var byteString = atob(dataURI.split(',')[1]);
// separate out the mime component
 var mimeString = dataURI.split(',')[0].split(':')[1].split(';')[0]

// write the bytes of the string to an ArrayBuffer
 var ab = new ArrayBuffer(byteString.length);
 var ia = new Uint8Array(ab);
 for (var i = 0; i < byteString.length; i++) {
  ia[i] = byteString.charCodeAt(i);
 }

// write the ArrayBuffer to a blob, and you're done

 return new Blob([ia], {type: mimeString});
};

Paster.prototype.findImageEl = function(paster) {

 if (paster.pasteCatcher.children.length > 0) {

  var dataURI = paster.pasteCatcher.firstElementChild.src;
  if (dataURI) {
   if (dataURI.indexOf('base64') === -1) {
    alert("Sorry, with Firefox you can only paste local screenshots and files. Use Chrome or IE11 if you need paster feature.");
    return;
   }

   var file = paster.dataURItoBlob(dataURI);
   paster.uploadFile(paster, file);
  }

  paster.pasteCatcher.innerHTML = '';

 } else
 {
  setTimeout(function() {
   paster.findImageEl(paster);
  }, 100);
 }
};


Complete example

I hope this is useful for you! It sure is useful for the Group-Office Content Management System and E-mail module! Here's the fully functional example that should work on any php server. Save it to pasteupload.php:


<?php
/**
 *
 * The MIT License (MIT)
 *
 * Copyright (c) 2014 Intermesh BV 
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 *
 */

if($_SERVER['REQUEST_METHOD']=='POST'){
 
 var_dump($_FILES);
 
 exit();
}
?>
<html>
<head>
<script type="text/javascript">

//Define paster object with contructor
var Paster = function(config) {

 for(var key in config){
  this[key]=config[key];
 }
 
 this.init();
};

Paster.prototype.pasteEl=null;

Paster.prototype.init=function() {

 var paster = this;

 if (window.Clipboard) {
  //IE11, Chrome, Safari
  this.pasteEl.onpaste=function(e){
   paster.handlePaste(paster, e);
  };
 } else
 {
  //On Firefox use the contenteditable div hack
  this.canvas = document.createElement('canvas');
  this.pasteCatcher = document.createElement("div");
  this.pasteCatcher.setAttribute("id", "paste_ff");
  this.pasteCatcher.setAttribute("contenteditable", "");
  this.pasteCatcher.style.cssText = 'opacity:0;position:fixed;top:0px;left:0px;';
  this.pasteCatcher.style.marginLeft = "-20px";
  document.body.appendChild(this.pasteCatcher);


  this.pasteEl.onblur=function(e) {
   paster.pasteCatcher.focus();
  };

  this.pasteCatcher.onpaste=function(e) {

   paster.findImageEl(paster);
  };
 }
};

 
Paster.prototype.dataURItoBlob=function(dataURI, callback) {
// convert base64 to raw binary data held in a string
// doesn't handle URLEncoded DataURIs - see SO answer #6850276 for code that does this

 console.log(dataURI);

 var byteString = atob(dataURI.split(',')[1]);
// separate out the mime component
 var mimeString = dataURI.split(',')[0].split(':')[1].split(';')[0]

// write the bytes of the string to an ArrayBuffer
 var ab = new ArrayBuffer(byteString.length);
 var ia = new Uint8Array(ab);
 for (var i = 0; i < byteString.length; i++) {
  ia[i] = byteString.charCodeAt(i);
 }

// write the ArrayBuffer to a blob, and you're done

 return new Blob([ia], {type: mimeString});
};

Paster.prototype.findImageEl = function(paster) {

 if (paster.pasteCatcher.children.length > 0) {

  var dataURI = paster.pasteCatcher.firstElementChild.src;
  if (dataURI) {
   if (dataURI.indexOf('base64') === -1) {
    alert("Sorry, with Firefox you can only paste local screenshots and files. Use Chrome or IE11 if you need paster feature.");
    return;
   }

   var file = paster.dataURItoBlob(dataURI);
   paster.uploadFile(paster, file);
  }

  paster.pasteCatcher.innerHTML = '';

 } else
 {
  setTimeout(function() {
   paster.findImageEl(paster);
  }, 100);
 }
};

Paster.prototype.processing = false; //some wierd chrome bug makes the paste event fire twice when using javascript prompt for the filename

Paster.prototype.handlePaste = function(paster, e) {

 //don't do this twice
 if (paster.processing) {
  return;
 }

 //loop through all clipBoardData items and upload it if it's a file.
 for (var i = 0; i < e.clipboardData.items.length; i++) {
  var item = e.clipboardData.items[i];
  if (item.kind === "file") {

   paster.processing = true;
   e.preventDefault();
   paster.uploadFile(paster, item.getAsFile());
  }
 }
};


Paster.prototype.uploadFile = function(paster, file) {

 var xhr = new XMLHttpRequest();

 //progress logging
 xhr.upload.onprogress = function(e) {
  var percentComplete = (e.loaded / e.total) * 100;

  console.log(percentComplete);
 };

 //called when finished
 xhr.onload = function() {
  if (xhr.status === 200) {
   alert("Sucess! Upload completed. PHP response will be put in the textarea.");
  } else {
   alert("Error! Upload failed");
  }

  paster.processing = false;
 };

 //error handling
 xhr.onerror = function() {
  alert("Error! Upload failed. Can not connect to server.");
 };

 //trigger a callback when it's successful
 xhr.onreadystatechange = function()
 {
  if (xhr.readyState === 4 && xhr.status === 200)
  {
   if (paster.callback) {
    paster.callback.call(paster.scope || paster, paster, xhr);
   }
  }
 };


 //prompt for the filename
 var filename = prompt("Please enter the file name", "Pasted image");

 if (filename) {

  //upload the file
  xhr.open("POST", "<?php echo $_SERVER['PHP_SELF']; ?>", {
   filename: filename,
   filetype: file.type
  });
  
  //send it as multipart/form-data
  var formData = new FormData();
  formData.append("pastedFile", file);
  xhr.send(formData);
 }
};
</script>
</head>
<body>
 
 <textarea id="text" style="width:500px;height:500px;">
 
On chrome paste a screenshot in here and it will be uploaded.

On firefox first click outside the text area and paste the screenshot with CTRL+V

 </textarea>
 
 
 <script>
 
 //Create the paster object and connect it to the textarea.
 var paster = new Paster({
  pasteEl:document.getElementById("text"),
  callback:function(paster, xhr){
   
   //put the response in the textarea
   paster.pasteEl.value=xhr.responseText;
  }
 });
 
 //focus the textarea
 paster.pasteEl.focus();
 
 </script>
 
</body>
</html>

Thursday, March 6, 2014

Synchronize your Android phone with Group-Office without using Microsoft ActiveSync

Synchronizing Group-Office Professional using Microsoft ActiveSync is the easiest to setup and works great. But some might need an alternative. For those who don't know Group-Office yet here's a screenshot of the calendar:





I myself used to own a Samsung Galaxy S2 and I always happily synchronized this with Group-Office using Microsoft ActiveSync. I recently got myself a new LG Nexus 5 and I immediately setup ActiveSync again. But I immediately missed two important features:
  1. No tasks synchronization anymore! The stock Android OS doesn't have a tasks app. 
  2. I couldn't search for older mail anymore. The Samsung Galaxy had a very nice feature to search for mail on the server. So even when you only sync two weeks of mail, you could still find an older message by keyword. 
So I started looking for alternatives:

E-mail with IMAP

The e-mail solution was easy. I just setup an IMAP account that works out of the box with Android and Group-Office. Here's a screenshot of the Android e-mail app:



Contacts and Calendar sync

I found “Caldav Sync Free Beta” and “CardDav Sync Free beta” in the Google play store that were both easy to setup and work great. They are open source and free of charge as well!

The apps are implemented as a sync adapter so they seamlessly integrate with the standard android contacts and calendar application. The screenshot below just shows the standard Android calendar because that's all you will see and that's just the way it should be!


Tasks

Synchronizing tasks with Group-Office proved to be more difficult. I tried “CalDAV Tasksync beta free” and it worked, but I think it's in an early stage of development because the interface is not very good yet. But it looks promising for the future.

I also tried some other tasks apps that didn't work at all. I then decided to remove “Caldav Sync Free Beta” and replace it with “CalDAV-Sync” because it also has a nice tasks app with it. This app costs € 2,59 but it's worth the money! It also integrates nicely with the android calendar app and the tasks application looks and works great too!

Here's a screenshot of the Tasks app that synchronizes with Group-Office:



Ending

In lots of cases synchronizing Group-Office with Microsoft ActiveSync will be easier as you only have to setup one account for all. But the CalDAV alternatives work great too. It's just that you have to enter your new password four times when you setup a new one ;)

But as I did my research on these apps I came across great open source projects that might be the base of a combined tasks, calendar and contacts sync app for Group-Office!

Thursday, February 20, 2014

Creating your own project and time tracking reports

Introduction

With the new projects version 2 module we have created a very powerful and flexible reporting system. We supply open source reports written in PHP so anybody with PHP knowledge can create their own reports.

The reports that come with Group-Office are located in “modules/projects2/report”. There are already reports about time tracking of employees, project information and a general overview of all projects.

All these reports extend a simple abstract class that holds some basic functionality.



Creating the PHP code

It's best to keep your own reports in your data folder (defined in $config['file_storage_path']='/home/groupoffice'). The projects module automatically detects reports in “/home/groupoffice/php/projects2/report”.

So if you would create a report called “MyReport” then you must create the file “MyReport.php” there. Remember that everything is case sensitive!

In MyReport.php define the class name like this:



The start “GOFS” in the class name will instruct the Group-Office class loader to look in the home folder for this report.

In the abstract class there are 3 methods that will control when this report template will be available:

  1. supportsSelectedProject
  2. supportsBatchReport
  3. supportedProjectTypes

It can show up when you run a batch report from the main toolbar in the module or when a project container or project was selected through the “New” → “Report” menu option.

In the following example we create an Excel report using the PHPExcel library.



<?php
/**
 *
 * The MIT License (MIT)
 *
 * Copyright (c) 2014 Intermesh BV 
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 *
 */

/**
 * Simple XLSX report for project information
 *
 */
class GO_Projects2_Report_MyReport extends GO_Projects2_Report_Abstract {

 /**
  * Return filename extension.
  * @return string
  */
 public function fileExtension() {
  return 'xlsx';
 }

 /**
  * Return the name of the report used in the drop down
  * 
  * @return string
  */
 public function name() {
  return 'My Excel report';
 }

 /**
  * With this enabled it will only show up if this report is created from the
  * "New" menu in a project.
  * 
  * @return boolean defaults to false
  */
 public function supportsSelectedProject() {
  return true;
 }

 /**
  * With this enables it shows up when you create a report from the main toolbar
  * in the projects module.
  * 
  * @return boolean
  */
 public function supportsBatchReport() {
  return false;
 }

 /**
  * Indicates which project types are accepted for this report.
  * @return boolean Array containing for example: GO_Projects2_Model_Template::PROJECT_TYPE_PROJECT or GO_Projects2_Model_Template::PROJECT_TYPE_CONTAINER
  */
 public function supportedProjectTypes() {
  return array(GO_Projects2_Model_Template::PROJECT_TYPE_PROJECT);
 }

 /**
  * Indicate whether this report supports a start and end date
  * 
  * @return boolean defaults to false
  */
 public function supportsDateRange() {
  return false;
 }

 /**
  * Indicate whether this report supports a start and end date
  * 
  * @return boolean defaults to false
  */
 public function supportsStatusFilter() {
  return false;
 }

 /**
  *
  * @var GO_Base_Util_Excel 
  */
 private $_xls;

 /**
  * Start rendering the report.
  * 
  * @param boolean $return to return the file data as string.
  */
 public function render($return = false) {


  $this->project->id;

  //Create an Excel object. See http://www.codeplex.com/PHPExcel) for more 
  //information
  $this->_xls = new GO_Base_Util_Excel();

  $this->_xls->getProperties()->setCreator(GO::user()->name);
  $this->_xls->getProperties()->setLastModifiedBy(GO::user()->name);

  $this->_xls->getProperties()->setTitle($this->name());
  $this->_xls->getProperties()->setSubject($this->project->path);

  $this->_xls->setActiveSheetIndex(0);
  $this->_xls->getActiveSheet()->setTitle("Project information");


  $this->_xls->setDefaultStyle('Arial', 12);
  $this->_xls->setDefaultWidth(15);

  $this->_xls->setCellValue('A1', GO::t('strName'));
  $this->_xls->setCellValue('B1', $this->project->path);


  $this->_xls->setCellValue('A2', GO::t('status', 'projects2'));
  $this->_xls->setCellValue('B2', $this->project->status->name);

  $this->_xls->setCellValue('A3', GO::t('startTime', 'projects2'));
  $this->_xls->setCellValue('B3', $this->project->getAttribute('start_time', 'formatted'));


  $this->_xls->setCellValue('A4', GO::t('projectDue_time', 'projects2'));
  $this->_xls->setCellValue('B4', $this->project->getAttribute('due_time', 'formatted'));

  $this->_xls->setCellValue('A5', GO::t('lag', 'projects2'));
  $this->_xls->setCellValue('B5', GO_Base_Util_Date::minutesToTimeString($this->project->getLag()));

  $this->_xls->setCellValue('A6', GO::t('strDescription'));
  $this->_xls->setCellValue('B6', $this->project->description);
  $this->_xls->getActiveSheet()->getStyle("B6:B6")->getAlignment()->setWrapText(true);


  $file = GO_Base_Fs_File::tempFile($this->filename, $this->fileExtension());


  $this->_xls->save($file->path());


  if ($return) {
   //we must return it as string   
   $content = $file->getContents();
   $file->delete();
   return $content;
  } else {

   //we output it to the browser
   GO_Base_Util_Http::outputDownloadHeaders($file);
   $file->output();
  }
 }

}


Ending

I hope this simple example gets you started with project reports. The existing PDF reports are also a great resource to help you get going.

Tuesday, February 11, 2014

A new interface for Group-Office

First I will share something about the history of the Group-Office interface.

2003
When I first started with an interface for Group-Office back in 2003, life was easy. I just had to worry about the desktop. There were no smart phones and tablets yet! I built my own interface library in pure PHP 4.4 and these rendered HTML tables, treeviews etc. Javascript was not really evolved yet so all rendering was done with PHP. Every page was a separate PHP file that used various classes to render different widgets.

2007
Javascript evolved and it became possible to build very rich interfaces. I decided to make a complete rewrite of Group-Office using Sencha ExtJS 2! ExtJS is a very rich javascript library that has many widgets. The interface was now completely javascript driven and Group-Office had the most sophisticated web interface around. The whole interface was one PHP page that loaded the javascripts. All data is loaded with AJAX calls.

Today...
We still use ExtJS 3.4. This version is outdated but upgrading is a pain. Group-Office has become a big application and upgrading to ExtJS 4.x would in fact mean a complete rewrite of the interface. This is in my humble opinion a major drawback of ExtJS. They tend to completely change their API making it very backwards incompatible. The 2 to 3 upgrade was doable since we didn't even complete the ExtJS 2 version of Group-Office.

Another approach to take in consideration is responsive design. ExtJS is intended for the desktop and the desktop only. It doesn't work well on tablets and smart phones. Today tablets and smart phones can't be ruled out anymore. So if we want to keep up to date we have two options:

  1. Build a new Extjs 4 interface and a Sencha Touch application for tablets and smart phones
  2. Move away from Sencha and build a new responsive framework that works on the Desktop, Tablet and smartphone.

Option 1: Upgrade ExtJS and use Sencha Touch

Pro's:
  1. ExtJS is mature and has all the widgets we would ever need.
  2. The interface looks consistent throughout all modules.
Con's
  1. Upgrading to new major releases of ExtJS has proven to be difficult
  2. We need to maintain two applications. The desktop (ExtJS) and mobile version (Sencha Touch)
  3. The learning curve for ExtJS is steep. It's hard for new developers to start with Group-Office development.
  4. Theming is tough

Option 2: Build a completely new responsive interface framework

Pro's:
  1. One application for both desktop and mobile devices
  2. Flexibility, we can build it anyway we like. We can make it easy to theme.
  3. It will be fun to create your own framework and our developers will get more satisfaction with their own creations.
Con's
  1. We have to develop a lot of widgets that ExtJS already has.
  2. More testing to be done with cross browser support.
  3. More documentation to be written


Conclusion

I don't have a conclusion yet. I hope this article will bring some discussion here and in our company.

I will also start to experiment with alternatives to ExtJS and build a simple responsive interface.

Friday, January 31, 2014

Install Group Office with 1-click using Installatron

We are proud to announce that Group-Office is now officially available from Installatron.

Installatron is a one-click web application installer that enables Group Office and other top web applications to be instantly installed and effortlessly managed. Get Group Office up and running on your website in seconds and discover just how easy Installatron makes it to manage web applications. Group Office installations managed by Installatron can updated (on-demand or automated), cloned, backed up and restored, edited to change installation parameters, and more.

Many web hosting providers provide Installatron through their web hosting control panel. If Installatron is not available from your provider, you can use Installatron directly from Installatron.com.

To install Group Office through your web hosting provider's control panel (if available):

* Login to your web host's control panel, navigate to "Installatron", click "Group Office", and choose the "Install this application" option.
* Change any of the install prompts to customize the install. For example, you can choose a different language for Group Office.
* Click the "Install" button to begin the installation process. You will be redirect to a progress page where you can watch as Group Office is installed within a few seconds to your website.

To install Group Office directly from Installatron.com:

* Navigate to Group Office @Installatron and choose the "Install this application" option.
* Enter your hosting account's FTP or SSH account information, and then enter MySQL database information for a created database. For increased security, create a separate FTP account and MySQL database for your Group Office installation.
* Change any of the install prompts to customize the install. For example, you can choose a different language for Group Office.
* Click the "Install" button to begin the installation process. You will be redirect to a progress page where you can watch as Group Office is installed within a few seconds to your website.

If you experience any problems or want to share your experience using Group Office and Installatron together, email Installatron at: feedback (at) installatron.com

Monday, January 27, 2014

Implementing PHP namespaces in an existing project

If you've been developing for a longer time you know the problem. You started coding a project and now PHP comes with this new cool feature. At first you want to stay backwards compatible with older versions but there comes a point where you want to use these new features. Otherwise your code will be outdated and less attractive to new developers.
Such a feature we didn't have were namespaces. Namespaces are very useful to organize code and to avoid naming conflicts when you include a vendor library. For example you can have a function called "sendmail()". Now you'll have a problem when you want to include a 3rd party mail library which uses the exact same function.

With Group-Office we solved this problem before PHP namespaces arrived by prefixing. For example a class in the address book module would be called "GO_Addressbook_Model_Contact". All of our classes where prefixed by "GO" and followed up by the module name. This way we hacked our namespaces in with underscores. This was invented before PHP 5.3 became mainstream. Now almost all servers run PHP 5.3 and higher so it's time to change to namespaces! But this can be a tricky task which such a large project as Group-Office. But because we used the underscore namespace hack it wasn't so difficult at all. I wrote up with a script that could convert our project in one GO! It turns

class GO_Addressbook_Model_Contact{} into:

<?php

namespace \GO\Addressbook\Model;

class Contact{

}


Some prerequisites

While writing this script I did face a couple of problems I had to resolve manually. We used some names that where incompatible. For example:

GO_Base_Util_Array would convert into:

<?php

namespace GO\Base\Util;


class Array{

}


Using "Array" as a class name is invalid in PHP so I had to refactor these names. I found out that refactoring these names before using the script was the best approach. This way we could use Netbeans to easily refactor these names.
The list of illegal names was:

  • Function
  • Interface
  • Abstract
  • Switch
  • List
There are probably more but these where the only ones used in Group-Office.

I also had to make sure all existing PHP classes that were used are prefixed with a backslash. For example "new PDO" had to be written as "new \PDO".

The script

This script assumes "GO" is a global static singleton class and all classes are written in the GO_Module_Type_Class form. It also requires one class per file. You could easily modify it to work for your project if you meet these requirements.


<?php
/**
 *
 * The MIT License (MIT)
 *
 * Copyright (c) 2014 Intermesh BV 
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 *
 */


//find all PHP files except updates.php and updates.inc.php because we shouldn't touch them
$cmd = 'find . -type f \( -iname "*.php" ! -iname "updates*" \);';
exec($cmd, $scripts, $return_var);

//return var should be 0 otherwise something went wrong
if($return_var!=0)
 exit("Find command did not run successfully.\n");

foreach($scripts as $script){
 
 //skip old files. We don't use .inc.php anymore in the new framework
 if(substr($script,-14)=='.class.inc.php' || in_array(basename($script),array('namespacer.php','action.php','json.php')))
  continue;
 
 
 
 //get the contents of the PHP Script
 $content = $oldContent = file_get_contents($script);
 
 //Our main global static function GO::function() is easiest to identify like this
 $content = str_replace('GO::', '\\GO::', $content);
 
 //All GO classes are build up like GO_Module_SomeClass so we can match them with
 //a regular expression.
 $regex = '/[^A-Za-z0-9_-](GO(_[A-Za-z0-9]+)+)\b/';
 
 $classes = array();
 
 if(preg_match_all($regex, $content, $matches))
 {
  
  
  //loop through the matched class names and store the old classname as key
  //and the new classname as value
  foreach($matches[1] as $className){
   
   //skip all uppercase classnames. They are old eg. GO_USERS, GO_LINKS
   if($className!=strtoupper($className)){ 
    if(!in_array($className, $classes)){
     $classes[$className]='\\'.str_replace('_','\\', $className);
    }
   }
  }
  
  //replace all old class names with the new namespaced ones.
  foreach($classes as $oldClassName=>$newClassName){
   $content = str_replace($oldClassName, $newClassName, $content);
  }
  
  //now we have a problem with the class declarations.  
  //we only have one class per file!
  foreach($classes as $oldClassName=>$newClassName){
   $classDeclarationRegex = '/(class|interface)\s('.preg_quote($newClassName,'/').')/';

            //Attempt to find a class definition in this file.
   if(preg_match($classDeclarationRegex,$content, $classDeclarationMatches,PREG_OFFSET_CAPTURE)){
    
    echo "Found ".$newClassName."\n";
    
    //strip last part of the class name to become the namespace.
    //eg. class; \GO\Email\Model\ImapMessageAttachment will have namespace:
    //GO\Email\Model
    $namespace = trim($newClassName,'\\');
    $lastBackSlashPos = strrpos($namespace,'\\');
    $namespace = substr($namespace,0, $lastBackSlashPos);
    

    //find place in the file to enter the "namespace GO\Email\Model;" declaration.
    //we can do this above the line with declaration "class ImapMessageAttachment"    
    $offset = $classDeclarationMatches[0][1];
    $lastLineBreakPos = strrpos(substr($content,0,$offset), "\n");
    
    $declaration = "\n\nnamespace ".$namespace.";\n\n";

                //Inset the declaration in the file content
    $firstPart = substr($content,0,$lastLineBreakPos);
    $lastPart = substr($content, $lastLineBreakPos);    
    $content = $firstPart.$declaration.$lastPart;
    
    //now we must remove the namespace from class usages in this file.
                //eg. \GO\Base\Db\ActiveRecord becomes ActiveRecord.
    $content = preg_replace('/([^"\'])\\\\'.preg_quote($namespace,'/').'\\\\/', "$1", $content);

   }
  }
  

 }
 
 //some doubles could have been made
 $content = str_replace('\\\\GO', '\\GO', $content);
 
 //if the contents were modified then write them to the file.
 if($oldContent != $content){
  echo "\nReplacing $script\n";
  file_put_contents($script, $content);
 }


    echo "All done!\n";
 
}

Friday, January 17, 2014

Rsync backup script for our web and MySQL servers

One of the most important matters for an IT company is taking care of backups. We make backups in twofold. We make full snapshots of our virtual machines in the data center and we make incremental backups to another location. This is absolutely necessary in case of a major disaster like theft or a fire. We keep 14 incremental backups so when a customer wants to recover a single file he deleted accidentally we can recover it easily as long as it didn't happen earlier than 14 days ago.

We used to have a script on each server that took care of the backup. With rsync and mysql dump. This worked well at first but as our number of servers increased it could happen that the process would fail due to multiple backups running at the same time. Multiple copy commands on the server made everything very slow. When you aborted a slow backup the incremental snapshot could only be half copied resulting in duplicate data being transferred the next backup. When this happpened I decided to turn the process around. I wrote a script that runs on the backup server that connects to each server one by one. The programming language I know best is PHP so I wrote it in that language.

Preparing the server that we need to backup

The server needs to support "rsync". I also want to backup MySQL so I also needed "mysqldump". "mysqldump" comes with the package "mysql-client-5.5" and is usually already installed.
I ran:

$ apt-get install rsync

That's all I needed to do!

Installing the backup server

The backup server can run any Linux distribution supporting PHP on the command line and rsync. I used SSH keys to connect to the servers.
In my case I installed a minimal Debian server with php and rsync on it. I ran:

$ apt-get install rsync php-cli

I used SSH to connect to the web servers with a private and public key pair. I generated the key pair with the following command:

$ ssh-keygen

The resulting key files are then located in /root/.ssh/id_rsa(.pub).

Adding a target server

Now it's time to connect to the target web server. For example. "web1.example.com". To install the public key I ran:

$ ssh-copy-id root@web1.example.com

I entered the root password so that my public key (/root/.ssh/id_rsa.pub) is added to the authorized keys on the web server.

I verified that this worked by logging in with the key. There's no need to enter a password:

$ ssh root@web1.example.com
$ exit

Now we have a backup server that can access our web server with public key authentication.

Installing the script

Put the script on the server. For example in /root/backup.php
Edit the following values on top of the script:
<?php

...


/**
 * The backup report will be mailed to this address.
 * 
 * @var String 
 */
private $emailTo = 'postmaster@example.com';

/**
 * The location of the private key to use for SSH
 * 
 * @var String 
 */
private $sshKey = '/root/.ssh/id_rsa'; 

/**
 * An array of server to backup
 * 
 * @var array 
 */
private $servers = array(
 array(
  'host' => 'web1.example.com', //hostname of target server to backup
  'user' => 'root', //the user on the target server.          
  'port' => 22, //SSH port
  'backupFolders' => array('/etc', '/home', '/var/www'), //folders to backup
  'targetFolder' => '/home/backup/web1', //folder on the backup server to store the backups
  'rotations' => 1, //number of rotations.
  'backupMysql' => true, //Backup MySQL database. Note that "mysqldump" must be installed on the target server,
  'mysqlUser' => 'root',
  'mysqlPassword' => 'secret',
  'mysqlBackupDir' => '/home/mysql_backup' //MySQL backup directory on the target server
 )
);
 
...




After configuration I ran a test:

$ php /root/backup.php

Then I scheduled it in a cron job:

$ crontab -e

I added this line to run it on midnight every day:

0 0 * * php /root/backup.php


I hope this little tutorial is useful! This is the full backup script:

<?php
/**
 * Backup script to backup multiple servers including MySQL support.
 * Author: Merijn Schering <info@intermesh.nl> 
 * 
 * 

The MIT License (MIT)

Copyright (c) 2014 Intermesh BV <info@intermesh.nl>

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
 *
 */

class backup{

 /**
  * The backup report will be mailed to this address.
  * 
  * @var String 
  */
 private $emailTo = 'postmaster@example.com';

 /**
  * The location of the private key to use for SSH
  * 
  * @var String 
  */
 private $sshKey = '/root/.ssh/id_rsa'; 

 /**
  * An array of server to backup
  * 
  * @var array 
  */
 private $servers = array(
   array(
     'host' => 'web1.example.com', //hostname of target server to backup
     'user' => 'root', //the user on the target server.          
     'port' => 22, //SSH port
     'backupFolders' => array('/etc', '/home', '/var/www'), //folders to backup
     'targetFolder' => '/home/backup/web1', //folder on the backup server to store the backups
     'rotations' => 1, //number of rotations.
     'backupMysql' => true, //Backup MySQL database. Note that "mysqldump" must be installed on the target server,
     'mysqlUser' => 'root',
     'mysqlPassword' => 'secret',
     'mysqlBackupDir' => '/home/mysql_backup' //MySQL backup directory on the target server
   )
 );
 
 
 private $logDir;
 private $sshCmd;
 private $dateFormat;
 private $logFile;
 private $errors=array();

 public function __construct() {

  //set the timezone to avoid date() notices from PHP
  date_default_timezone_set('europe/amsterdam');

  //Make sure files are created with the right permissions
  umask(000);

  $old_error_handler = set_error_handler(array($this, "errorHandler"));

  $this->dateFormat = 'Y-m-d_H-i-s';

  if (!isset($this->logDir)) {
   $this->logDir = dirname(__FILE__) . '/log/';
  }
  $this->logFile = $this->logDir . '/' . date($this->dateFormat) . '.log';

  $this->execCommand('mkdir -p ' . $this->logDir);
 }

 /**
  * Executes command line program
  * 
  * @param stirng $cmd
  * @param boolean $halt
  * @return mixed false on failure or command output
  */
 private function execCommand($cmd, $halt = true) {
  exec($cmd, $output, $ret);
  if ($ret != 0) {

   $this->error("Failed to run command: '" . $cmd . "'");
   $this->info(implode("\n", $output));


   if ($halt) {
    $this->exitBackup();
   } else {
    return false;
   }
  } else {
   return $output;
  }
 }

 /**
  * Executes command on the remote server.
  * 
  * @param string $cmd
  * @return mixed false on failure or command output
  */
 private function execRemoteCommand($cmd, $halt=true) {

  $cmd = $this->sshCmd . ' "' . str_replace('"', '\"', $cmd) . '"';

  return $this->execCommand($cmd, $halt);
 }

 /**
  * Exits the backup  script and sends backup report
  * @param string $error
  */
 private function exitBackup($error = '') {
  if ($error != ''){
   $this->error($error);
  }

  $subject = "Backup report (success)";
  $body="";
  
  if(count($this->errors)){
   $subject = "Backup report (ERRORS!)";
   
   $body = "Error summary: \n".implode("\n---\n", $this->errors);
  }else
  {
   $subject = "Backup report (success)";
  }
  
  
  $body .= file_get_contents($this->logFile);
  
  $header = "From: Backup server \r\n";

  mail($this->emailTo, $subject, $body, $header);

  exit();
 }

 /**
  * Writes log message
  * 
  * @param string $text
  */
 private function info($text) {
  $text = '[' . date('Y-m-d H:i') . '] ' . $text . "\n";

  echo $text;

  file_put_contents($this->logFile, $text, FILE_APPEND);
 }
 
 
 /**
  * Writes error message
  * 
  * @param string $text
  */
 private function error($text) {
  $text = '[' . date('Y-m-d H:i') . '] !!ERROR!! :' . $text . "\n";

  echo $text;
  
  $this->errors[]=$text;

  file_put_contents($this->logFile, $text, FILE_APPEND);
 }

 private function setSshCmd($server) {
  $this->sshCmd = 'ssh -i ' . $this->sshKey . ' ' . $server['user'] . '@' . $server['host'] . ' -p ' . $server['port'];

  $this->info("Testing SSH connection");
  if($this->execCommand($this->sshCmd . ' -o "PasswordAuthentication=no" "exit"' , false)!==false){
   $this->info("Test OK");
  }else
  {
   $this->error("SSH test for ".$server['host']." failed!");
   return false;
  }


  $this->sshCmd .= ' -o "ServerAliveInterval 60"';

  return true;
 }

 /**
  * Runs the backup
  */
 public function run() {

  //We never want this script to run twice at the same time
  $lockFile = dirname(__FILE__) . '/backup.lock';
  if (file_exists($lockFile)) {
   $this->exitBackup("Lock file '" . $lockFile . "' exists. Please check last backup! It failed or it is still running.");
  } else {
   touch($lockFile);
  }

  foreach ($this->servers as $server) {


   $this->info("\n\n\n----------------------------------------------------\n\n\n");
   $this->info("Backing up " . $server['host']);

   if($this->updateLatest){
    $server['backupMysql']=false;
    $server['rotations']=1;
   }

   if(!$this->setSshCmd($server)){
    continue;
   }

   //Create the backup folder
   $this->execCommand('mkdir -p ' . $server['targetFolder']);


   //Delete the oldest backup
   $ls = $this->execCommand('ls -dXr ' . $server['targetFolder'] . '/*/', false);

   $newestBackup = isset($ls[0]) ? $ls[0] : false;

   if ($server['rotations'] > 1) {
    $oldBackupIndex = $server['rotations'] - 1;
    if (isset($ls[$oldBackupIndex])) {
     $this->info("Deleting oldest backup: " . $ls[$oldBackupIndex]);
     $this->execCommand('rm -Rf ' . $ls[$oldBackupIndex]);
     $this->info("Delete done");
    }
   }

   //Make a copy using hard links so we don't duplicate in diskspace
   $backupDate = date($this->dateFormat);
   $currentBackupDir = $server['targetFolder'] . '/' . $backupDate;

   if ($newestBackup) {
    if ($server['rotations'] > 1) {
     $this->info("Copying all from " . $newestBackup . " to " . $currentBackupDir);
     $this->execCommand('cp -al ' . $newestBackup . ' ' . $currentBackupDir);
    } else {
     $this->info("Moving previous backup dir from " . $newestBackup . " to " . $currentBackupDir);
     $this->execCommand("mv " . $newestBackup . " " . $currentBackupDir);
    }
   } else {
    $this->info("Creating first backup directory " . $currentBackupDir);
    $this->execCommand('mkdir ' . $currentBackupDir);
   }

   if (!is_writable($currentBackupDir)) {
    $this->exitBackup("Error: backup directory $currentBackupDir is not writable!");
   }


   $this->backupMysql($server);


   //Now run rsync for each backup source
   foreach ($server['backupFolders'] as $remoteBackupFolder) {
    $this->info("Backing up " . $remoteBackupFolder);

    $localBackupFolder = $currentBackupDir . '/' . $remoteBackupFolder;
    $this->execCommand('mkdir -p ' . $localBackupFolder);

    $cmd = 'rsync -av --delete -e "ssh -i ' . $this->sshKey . ' -p' . $server['port'] . '" ' . $server['user'] . '@' . $server['host'] . ':' . $remoteBackupFolder . '/ ' . $localBackupFolder . '/';
    $this->info("Running " . $cmd);
    //system($cmd, $ret);
    //run the rsync command and read the response line by line
    $fp = popen($cmd, "r");
    while (!feof($fp)) {
     // send the current file part to the browser 
     $this->info(fread($fp, 1024));
    }
    $ret = pclose($fp);

    if ($ret != 0) {
    
     if($ret == 24){
      $this->info("Some source files vanished during transfer. This is probably normal because the system is live.");
     }else{    
      $this->error("Failed running rsync command!\ncmd: ".$cmd."\nReturn code: ".$ret);
     }
    }
   }

   $this->info("Done with " . $server['host']);

   $this->info("\n\n\n----------------------------------------------------\n\n\n");

   //$contents = ob_get_contents(); 
  }

  //remove lock file
  unlink($lockFile);

  $this->exitBackup();
 }

 private function backupMysql($server) {
  if ($server['backupMysql']) {

   $this->info("Starting MySQL backup");

   if (empty($server['mysqlBackupDir'])) {
    $this->error("Please set " . $server['mysqlBackupDir']);
    return false;
   }
   if($this->execRemoteCommand('mkdir -p ' . $server['mysqlBackupDir'], false)===false){
    return false;
   }

   $cmd = 'mysql --user=' . $server['mysqlUser'] . ' --password=' . $server['mysqlPassword'] . ' -e "SHOW DATABASES;" | tr -d "| " | grep -v Database';
   //$this->info($cmd);
   $databases = $this->execRemoteCommand($cmd, false);

   foreach ($databases as $database) {
    if ($database != "information_schema" && $database != 'performance_schema') {
     $this->info("Dumping $database");
     $cmd = 'mysqldump --force --opt --user=' . $server['mysqlUser'] . ' --password=' . $server['mysqlPassword'] . ' --databases ' . $database . ' > "' . $server['mysqlBackupDir'] . '/' . $database . '.sql"';

     if($this->execRemoteCommand($cmd, false)===false){
      continue;
     }

     $this->info("Compressing " . $database);
     $cmd = "tar -C " . $server['mysqlBackupDir'] . " -czf " . $server['mysqlBackupDir'] . "/$database.tar.gz $database.sql";
     if($this->execRemoteCommand($cmd, false)===false){
      continue;
     }

     $cmd = 'rm ' . $server['mysqlBackupDir'] . "/$database.sql";
     if($this->execRemoteCommand($cmd, false)===false){
      continue;
     }



     //break;
    }
   }
   $this->info("MySQL backup done");
  } else {
   $this->info("MySQL backup NOT enabled");
  }
 }

 public function errorHandler($errno, $errstr, $errfile, $errline) {
  switch ($errno) {
   case E_USER_ERROR:
    // Send an e-mail to the administrator
    $msg = "!!!!! Error: $errstr \n Fatal error on line $errline in file $errfile";

    // Write the error to our log file
    $msg = "!!!!! Error: $errstr \n Fatal error on line $errline in file $errfile";
    break;

   case E_USER_WARNING:
    // Write the error to our log file
    $msg = "!!!!! Warning: $errstr \n in $errfile on line $errline";
    break;

   case E_USER_NOTICE:
    // Write the error to our log file
    $msg = "!!!!! Notice: $errstr \n in $errfile on line $errline";
    break;

   default:
    // Write the error to our log file
    $msg = "!!!!! Unknown error [#$errno]: $errstr \n in $errfile on line $errline";
    break;
  }

  $this->error($msg);

  // Don't execute PHP's internal error handler
  return TRUE;
 }

}

$backup = new backup();
$backup->run();