/* CreateSkeleton
 * Creates a skeleton PlayStation dev project based on the output of SymDumpTE
 * or another source of the appropriate JSON data
 * Copyright 2019 Ben Lincoln
 * https://www.beneaththewaves.net/
 * 
 * This file is part of CreateSkeleton.
 * 
 * CreateSkeleton is free software: you can redistribute it and/or modify
 * it under the terms of version 3 of the GNU General Public License as published by
 * the Free Software Foundation.
 * 
 * CreateSkeleton is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public License
 * along with CreateSkeleton (in the file LICENSE.txt).  
 * If not, see <http://www.gnu.org/licenses/>.
 */
// %PROJECT_NAME%: Machine-generated array-identifying script

import java.io.*;
import java.util.*;
import java.nio.charset.StandardCharsets;

import ghidra.app.plugin.core.script.Ingredient;
import ghidra.app.plugin.core.script.IngredientDescription;
import ghidra.app.script.GatherParamPanel;
import ghidra.app.script.GhidraScript;
import ghidra.app.services.DataTypeManagerService;
import ghidra.framework.plugintool.PluginTool;
import ghidra.program.flatapi.FlatProgramAPI;
import ghidra.program.model.address.*;
import ghidra.program.model.block.*;
import ghidra.program.model.data.*;
import ghidra.program.model.listing.*;
import ghidra.program.model.mem.*;
import ghidra.program.model.lang.*;
import ghidra.program.model.symbol.*;

import utilities.util.FileUtilities;

public class %PROJECT_NAME%TDRAggressiveArrayIdentification extends GhidraScript implements Ingredient
{	
	public int MaximumNonPrintableCharactersInCharArray = 1;
	
	protected void PrintDebugMessage(String msg)
	{
		//println("Debug: " + msg);
	}
	
	protected void PrintInfoMessage(String msg)
	{
		println("Info: " + msg);
	}
	
	// Ghidra represents C chars using the Java Character class instead of bytes, 
	// This basically promots them to the 0x00-0xFF range of Unicode.
	// This method is a workaround to basically get a byte back.
	// They're handled as ints to avoid a situation where unexpectedly the data is 
	// actually Unicode
	protected int GetCharNumFromData(Data d)
	{
		if (d.getValue() == null)
		{
			return 0;
		}
		Character valueChar = (Character)d.getValue();		
		int result = (int)valueChar;
		PrintDebugMessage("calculated char num " + String.valueOf(result) + " for input char '" + valueChar + "'.");
		if (result > 255)
		{
			println("Error: got a value greater than 0xFF for char at " + longAsHexString(d.getAddress().getOffset()) + ": " + String.format("0x%0X", result));
		}
		return result;
	}
	
	protected int[] GetGhidraCharArrayAsCharNumAray(List<Data> charArray)
	{
		int[] result = new int[charArray.size()];
		for (int i = 0; i < charArray.size(); i++)
		{
			result[i] = GetCharNumFromData(charArray.get(i));
		}
		return result;
	}
	
	protected int GetCountOfNonPrintableCharactersInCharArray(List<Data> charArray)
	{
		int result = 0;
		int[] arrayChars = GetGhidraCharArrayAsCharNumAray(charArray);
		
		for (int i = 0; i < arrayChars.length; i++)
		{
			boolean ignore = false;
			// ignore non-printable formatting characters
			// newline
			if (arrayChars[i] == 0x0A)
			{
				ignore = true;
			}
			// carriage return
			if (arrayChars[i] == 0x0D)
			{
				ignore = true;
			}
			// tab
			if (arrayChars[i] == 0x09)
			{
				ignore = true;
			}
			
			if (!ignore)
			{
				if ((arrayChars[i] < 0x20) || ((arrayChars[i] > 0x7E) && (arrayChars[i] <= 0xFF)))
				{
					result++;
				}
			}
		}
		return result;
	}

	protected int GetIndexOfNullByteInCharArray(List<Data> charArray, int startOffset)
	{
		int result = 0;
		int[] arrayChars = GetGhidraCharArrayAsCharNumAray(charArray);
		
		for (int i = startOffset; i < arrayChars.length; i++)
		{
			if (arrayChars[i] == 0x00)
			{
				return i;
			}
		}
		return -1;
	}
	
	protected int GetIndexOfFirstNullByteInCharArray(List<Data> charArray)
	{
		return GetIndexOfNullByteInCharArray(charArray, 0);
	}
	
	@Override
	public void run() throws Exception {
		IngredientDescription[] ingredients = getIngredientDescriptions();
		for (IngredientDescription ingredient : ingredients) {
			state.addParameter(ingredient.getID(), ingredient.getLabel(), ingredient.getType(),
				ingredient.getDefaultValue());
		}
		if (!state.displayParameterGatherer("Script Options")) {
			return;
		}
		
		String[] charTypeNameArray = new String[] { "char", "uchar", "unsigned char" };
		List<String> charTypeNames = Arrays.asList(charTypeNameArray);
		String[] stringTypeNameArray = new String[] { "string" };
		List<String> stringTypeNames = Arrays.asList(stringTypeNameArray);
		
		long minAddress = Long.decode(String.valueOf(state.getEnvironmentVar("MinimumAddress")));
		long maxAddress = Long.decode(String.valueOf(state.getEnvironmentVar("MaximumAddress")));
		int arrayDetectionMode = (Integer)state.getEnvironmentVar("ArrayDetectionMode");
		int charArrayHandlingMode = (Integer)state.getEnvironmentVar("CharArrayHandlingMode");
		int numPasses = (Integer)state.getEnvironmentVar("NumPasses");
		
		SymbolTable symbolTable = currentProgram.getSymbolTable();
		
		// for use in char array conversion
		DataType stringDT = null;
		try
		{
			stringDT = findDataTypeByName("string");
		}
		catch (Exception e)
		{
			stringDT = null;
		}
		if (stringDT == null)
		{
			stringDT = new StringDataType();
		}
		
		int totalArraysCreatedCount = 0;
		int totalCharArraysToStringsCount = 0;
		for (int passNum = 0; passNum < numPasses; passNum++)
		{
			int passArraysCreatedCount = 0;
			int passCharArraysToStringsCount = 0;
			if (numPasses > 1)
			{
				println("Beginning pass number " + String.valueOf(passNum + 1));
			}
			List<Data> currentArray;
			currentArray = new ArrayList<Data>();
			DataType currentArrayDataType = null;
			Address currentArrayStartAddress = null;
			int currentArrayDataTypeLength = -1;
			long previousDataOffset = -1;
			long previousDataLength = -1;
			
			Data data = getFirstData();

			while (true)
			{
				if (monitor.isCancelled())
				{
					break;
				}

				if (data == null)
				{
					break;
				}

				String currentStep = "getting the address of the current data to be processed";
				try
				{
					long dataOffset = data.getAddress().getOffset();
					
					currentStep = "determining if the address " + longAsHexString(dataOffset) + " is within the region to be processed";
					
					if ((dataOffset >= minAddress) && (dataOffset <= maxAddress))
					{
						currentStep = "getting the data type for the data at " + longAsHexString(dataOffset);
						DataType dt = data.getDataType();
						String currentDataDesc = " the " + dt.getName() + " at " + longAsHexString(dataOffset);
						currentStep = "determining if" + currentDataDesc + " is a continuation of a previous array";
						
						boolean isContinuationOfArray = true;
						if ((previousDataOffset == -1) || (previousDataLength == -1))
						{
							isContinuationOfArray = false;
							PrintDebugMessage("determined that" + currentDataDesc + " is not a continuation of an array, because there is no offset and/or length defined for previous data.");
						}
						else
						{
							if (previousDataLength != -2)
							{
								if (dataOffset != (previousDataOffset + previousDataLength))
								{
									isContinuationOfArray = false;
									PrintDebugMessage("determined that" + currentDataDesc + " is not a continuation of an array, because the previous data ended at " + longAsHexString(previousDataOffset) + " and had a length of " + longAsHexString(previousDataLength) + ", meaning that an additional array member should have begun at " + longAsHexString(previousDataOffset + previousDataLength) + ", but the current data begins at " + longAsHexString(dataOffset) + ".");
								}
							}
						}
						if (isContinuationOfArray && (stringTypeNames.contains(dt.getName())))
						{
							isContinuationOfArray = false;
							PrintDebugMessage("ignoring" + currentDataDesc + " because C arrays cannot contain variable-length elements.");
						}
						if (isContinuationOfArray && ((currentArrayDataType == null) || (dt.getName() != currentArrayDataType.getName())))
						{
							isContinuationOfArray = false;
							String dtName = "null";
							if (currentArrayDataType != null)
							{
								dtName = currentArrayDataType.getName();
							}
							PrintDebugMessage("determined that" + currentDataDesc + " is not a continuation of an array, because it has the data type '" + dt.getName() + "' instead of " + dtName + ".");
						}
						if (isContinuationOfArray)
						{
							// in "intelligent" mode, stop including array elements if one is encountered which has references to it
							// and also if it's explicitly labeled
							if (arrayDetectionMode == 1)
							{
								Reference[] refsTo = getReferencesTo(data.getAddress());
								if (refsTo.length > 0)
								{
									isContinuationOfArray = false;
									PrintDebugMessage("determined that" + currentDataDesc + " is not a continuation of an array, because there are " + String.valueOf(refsTo.length) + " reference(s) to it.");
								}
								if (isContinuationOfArray)
								{
									Symbol[] addressSymbols = symbolTable.getSymbols(data.getAddress());
									if (addressSymbols.length > 0)
									{
										isContinuationOfArray = false;
										PrintDebugMessage("determined that" + currentDataDesc + " is not a continuation of an array, because it has " + 		String.valueOf(addressSymbols.length) + " symbol(s).");
									}
								}
							}
						}
						
						currentStep = "determining if the current list of data should be defined as an array";
						
						boolean redefineCurrentArray = true;
						boolean discardCurrentArray = false;
						if (currentArray.size() < 2)
						{
							redefineCurrentArray = false;
							PrintDebugMessage("not defining the current list of data as an array because it has " + String.valueOf(currentArray.size()) + " member(s).");
						}
						if (redefineCurrentArray && isContinuationOfArray)
						{
							redefineCurrentArray = false;
							PrintDebugMessage("not defining the current list of data as an array because the current data is a new member of the same array.");
						}
						
						if (redefineCurrentArray)
						{
							int numElements = currentArray.size();
							int newArrayLength = numElements * currentArrayDataType.getLength();
							String arrayDesc = "defining an array of type '" + currentArrayDataType.getName() + "' and count " + String.valueOf(numElements) + " (total size: " + longAsHexString(newArrayLength) + ") at " + longAsHexString(currentArrayStartAddress.getOffset()) + ".";
							currentStep = "determining whether to handle the current list of data as a string";							
							boolean handledAsString = false;
							// 0 == do not convert any char arrays to strings
							if (charArrayHandlingMode > 0)
							{
								if (charTypeNames.contains(currentArrayDataType.getName()))
								{
									currentStep = "determining if special handling should be applied to a char array";
									boolean convertToString = false;
									int[] arrayChars = GetGhidraCharArrayAsCharNumAray(currentArray);
									if (arrayChars[arrayChars.length - 1] != 0)
									{
										PrintDebugMessage("not converting the char array at " + longAsHexString(currentArrayStartAddress.getOffset()) + " to a string because it does not end in a null byte.");
										convertToString = false;
									}
									else
									{
										// 1 == convert all char arrays to strings
										if (charArrayHandlingMode == 1)
										{
											PrintDebugMessage("converting the current char array to a string because the operating mode specifies all char arrays should be converted to strings.");
											convertToString = true;
										}
										// 2 == intelligent handling
										if (charArrayHandlingMode == 2)
										{
											int numNonPrintableChars = GetCountOfNonPrintableCharactersInCharArray(currentArray);
											if (numNonPrintableChars > MaximumNonPrintableCharactersInCharArray)
											{
												PrintDebugMessage("not converting the the char array at " + longAsHexString(currentArrayStartAddress.getOffset()) + " to a string because it has " + String.valueOf(numNonPrintableChars) + " non-printable character(s), above the threshold of " + String.valueOf(MaximumNonPrintableCharactersInCharArray) + ".");
												convertToString = false;
											}
											else
											{
												convertToString = true;
											}
										}
										if (convertToString)
										{
											long currentStringOffset = currentArrayStartAddress.getOffset();
											int currentCharArrayOffset = 0;
											int currentNullByteIndex = GetIndexOfFirstNullByteInCharArray(currentArray);
											int currentStringLength = (currentNullByteIndex + 1) - currentCharArrayOffset;
											boolean continueProcessing = true;
											while (continueProcessing)
											{
												if (monitor.isCancelled())
												{
													break;
												}
												currentStep = "converting the char array at " + longAsHexString(currentStringOffset) + " with length " + longAsHexString(currentStringLength) + " to a string.";
												PrintInfoMessage(currentStep);
												DataUtilities.createData​(currentProgram, currentArrayStartAddress, stringDT, currentStringLength, false, DataUtilities.ClearDataMode.CLEAR_ALL_CONFLICT_DATA);
												currentStringOffset = currentStringOffset + currentStringLength + 1;
												currentCharArrayOffset = currentCharArrayOffset + currentStringLength + 1;
												currentStringLength = (currentNullByteIndex + 1) - currentCharArrayOffset;
												if (currentNullByteIndex < (currentArray.size() - 1))
												{
													currentNullByteIndex = GetIndexOfNullByteInCharArray(currentArray, currentCharArrayOffset);
												}
												else
												{
													continueProcessing = false;
												}
												if ((currentNullByteIndex == -1) || (currentStringLength == -1))
												{
													continueProcessing = false;
												}
												passArraysCreatedCount++;
												passCharArraysToStringsCount++;
												if (continueProcessing)
												{
													PrintInfoMessage("processing additional string found at index " + String.valueOf(currentCharArrayOffset) + " in the original char array.");
												}
											}
											handledAsString = true;
										}
									}
								}
							}
							if (!handledAsString)
							{
								currentStep = arrayDesc;
								ArrayDataType arrayDT = new ArrayDataType(currentArrayDataType, numElements, currentArrayDataType.getLength());
								PrintInfoMessage(arrayDesc);
								DataUtilities.createData​(currentProgram, currentArrayStartAddress, arrayDT, newArrayLength, false, DataUtilities.ClearDataMode.CLEAR_ALL_CONFLICT_DATA);
								passArraysCreatedCount++;
							}
							discardCurrentArray = true;
						}
						else
						{
							if (!isContinuationOfArray)
							{
								discardCurrentArray = true;
								PrintDebugMessage("discarding the current list of data because the current data is not a new member of the same array.");
							}
						}
						
						if (discardCurrentArray)
						{
							currentArray = new ArrayList<Data>();
							currentArrayDataType = dt;
							currentArrayStartAddress = data.getAddress();
						}
						currentArray.add(data);
						
						previousDataOffset = dataOffset;
						previousDataLength = dt.getLength();
						if ((previousDataLength > 0x10000000) || (previousDataLength == -1))
						{
							previousDataLength = -2;
						}
					}		
				}
				catch (Exception e)
				{
					println(e.toString() + " thrown while " + currentStep);
				}
				data = getDataAfter(data);
			}
			if (numPasses > 1)
			{
				PrintInfoMessage("created " + String.valueOf(passArraysCreatedCount) + " arrays in pass " + String.valueOf(passNum + 1) + ".");
				PrintInfoMessage("converted " + String.valueOf(passCharArraysToStringsCount) + " char arrays to strings in pass " + String.valueOf(passNum + 1) + ".");
			}
			totalArraysCreatedCount += passArraysCreatedCount;
			totalCharArraysToStringsCount += passCharArraysToStringsCount;
			
		}
		PrintInfoMessage("created " + String.valueOf(totalArraysCreatedCount) + " total arrays.");
		PrintInfoMessage("converted a total of " + String.valueOf(totalCharArraysToStringsCount) + " char arrays to strings.");
		//println("Done!");
	}
	
	protected String longAsHexString(long addr)
	{
		return String.format("0x%08X", addr);
	}
	
	// from PrintStructureScript.java
	private DataType findDataTypeByName(String name)
	{
		PluginTool tool = state.getTool();
		DataTypeManagerService service = tool.getService(DataTypeManagerService.class);
		DataTypeManager[] dataTypeManagers = service.getDataTypeManagers();
		for (DataTypeManager manager : dataTypeManagers)
		{
			DataType dataType = manager.getDataType(name);
			if (dataType != null)
			{
				return dataType;
			}
		}
		return null;
	}
	
	@Override
	public IngredientDescription[] getIngredientDescriptions()
	{
		// note: as of this writing, trying to use GatherParamPanel.ADDRESS results in a null pointer exception when reading the value
		IngredientDescription[] retVal = new IngredientDescription[]
		{
			new IngredientDescription("ArrayDetectionMode", "Arrays (0 = All Consecutive, 1 = Intelligent)", GatherParamPanel.INTEGER, 1),
			new IngredientDescription("CharArrayHandlingMode", "Char Array Handling (0 = Leave As-Is, 1 = Always Convert to Strings, 2 = Intelligent)", GatherParamPanel.INTEGER, 2),
			new IngredientDescription("MinimumAddress", "Minimum Address for Initial Pass", GatherParamPanel.STRING, "%PS_PRIMARY_MEMORY_BEGIN%"),
			new IngredientDescription("MaximumAddress", "Maximum Address for Initial Pass", GatherParamPanel.STRING, "%PS_PRIMARY_MEMORY_END%"),
			new IngredientDescription("NumPasses", "Number of Passes", GatherParamPanel.INTEGER, 1)
		};
		return retVal;
	}
	
}
