How Can We Help?

< All Topics

Imagium when clubbed with web crawlers can do real wonders. For larger projects It can save coding effort worth a few weeks.

Web crawlers can crawl through all the public pages of your website and those pages can serve as a baseline and the subsequent run’s can be compared against this baseline.

package Crawlers.copy;

import java.util.ArrayList;
import org.apache.commons.codec.binary.Base64;
import org.json.simple.JSONObject;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import io.restassured.RestAssured;
import io.restassured.response.Response;
import io.restassured.specification.RequestSpecification;
import org.openqa.selenium.JavascriptExecutor;
import org.openqa.selenium.OutputType;
import org.openqa.selenium.TakesScreenshot;
import org.openqa.selenium.WebDriver;

public class Crwalers {

	static WebDriver driver;
	static int linkcount = 0; 
	//To scan maximum number of pages set maxCount = -1;
	static int maxCount = 20;
	static boolean exitLoop = false;
	static String uid = "";
	//Set project Key
	static String projectKey = "de8a3afc-9f58-440c-a424-3251bd05d9d9";
	//Set the Seed URL
	static String urlSeed ="";
	//Set filter for the child pages, else keep it blank 
	static String filter = "";
	//Set Test Name
	static String testName = "Codeless Automation";
	//Set End points
	static String getUIDEndPoint ="";
	static String validateEndPoint = "";

	public static void main(String args[]) throws Exception
		//Using selenium Launch browser and navigate to URL
		System.setProperty("", "C:\\drivers\\chromedriver.exe");
		driver = new ChromeDriver();
		//driver.manage().window().setSize(new Dimension(1024, 768));
			uid = getUID(testName, projectKey);					
			crawl(1,urlSeed, new ArrayList<String>());
		//	linkcount
			System.out.println("TotalCount: " + Integer.toString(linkcount));	

	public static void crawl(int level, String url, ArrayList<String> visited) throws IOException 
			Document doc = request(url, visited);
			if (doc !=null) {
				for(Element link :"a[href]"))
					String next_link = link.absUrl("href");
							if (visited.contains(next_link)==false)
									crawl(level++, next_link, visited);
	private static Document request(String url, ArrayList<String> v)
		try {
			Connection con = Jsoup.connect(url);
			Document doc = con.get();
				linkcount += 1;
				if( maxCount>0)
					if (linkcount  > maxCount-1)
						exitLoop = true;
				//driver.manage().window().setSize(new Dimension(1024, 768));
			    JavascriptExecutor js = (JavascriptExecutor)driver; 
			    js.executeScript("return = 'hidden';");
				String scrBase64 = ((TakesScreenshot) driver).getScreenshotAs(OutputType.BASE64);
				postRequest(url.replace("#","_"), uid, scrBase64);
				System.out.println("Link: "+url);
				return doc;
			return null;
		catch(Exception e) {
			return null;
	private static String encodeFileToBase64Binary(File file){
        String encodedfile = null;
        try {
			FileInputStream fileInputStreamReader = new FileInputStream(file);
            byte[] bytes = new byte[(int)file.length()];
            encodedfile = new String(Base64.encodeBase64(bytes), "UTF-8");
        } catch (FileNotFoundException e) {
        } catch (IOException e) {
        return encodedfile;
	//Get unique Test ID for a specific project using Rest Assured 
	public static String getUID(String testName, String projectKey) {
		try {
			RequestSpecification request = RestAssured.given();
			request.header("content-type", "application/json");
			JSONObject json = new JSONObject();
			json.put("TestName", testName);
			json.put("ProjectKey", projectKey);
			json.put("Mode", "Default");

			Response response = request.when().post(getUIDEndPoint);
			int code = response.getStatusCode();
			String response_id = response.getBody().asString();
			System.out.println("TestID: " + response_id);
			return response_id;
		} catch (Exception ex) {
			return ex.toString();

	//Post a request using
	public static void postRequest(String stepName, String uid, String imagebase64) throws IOException {
		RequestSpecification request1 = RestAssured.given();
		request1.header("content-type", "application/json");
		JSONObject jo = new JSONObject();
		jo.put("TestRunID", uid.replace("\"", ""));
		jo.put("StepName", stepName);
		jo.put("ImageBase64", imagebase64);
		System.out.println("imagebase64:" + imagebase64);
		Response response1 = request1.when().post(validateEndPoint);
		String response_id1 = response1.getBody().asString();
		System.out.println("Response: " + response_id1);