MazeHunter
For the Mobile Development semester in school I’ve worked on an android app. MazeHunter is a drawing game for children with dyslexia. It trains the brain hemispheres in a fun and playful way.
The app is played on a tablet, with a whiteboard in front of it to draw on. The tablet has a mirror in front of the camera so that it is able to capture the whiteboard.
The previous group of students used an image processing library to be able to detect the drawn shapes and cut them from the background.
I’ve worked on an app that is used test and tweak the image processing steps. Previously the code was stuffed in a single function. I’ve split the function up in separate steps and made the variables of each step modifiable in the app.
class OpenCVActivity : AppCompatActivity(), IManageVariables
{
private lateinit var binding: ActivityOpencvBinding
private var selectedOperation: Int = 0
private var nOperations: Int = 0
private var variables = Array(7){ mutableMapOf<String, AppVariable>() }
private var appVariables = arrayListOf<AppVariable>()
private val appVariableAdapter = AppVariableAdapter(appVariables, this)
private var images : ArrayList<Bitmap> = ArrayList()
private var mats : ArrayList<Mat> = ArrayList()
private var selectedImageIndex = 0
private lateinit var operationsDropdown: Spinner
private lateinit var resetButton: Button
private lateinit var applyButton: Button
private lateinit var undoButton: Button
private lateinit var rawInputImage: Bitmap
private lateinit var currentImage: Mat
override fun onCreate(savedInstanceState: Bundle?)
{
super.onCreate(savedInstanceState)
binding = ActivityOpencvBinding.inflate(layoutInflater)
setContentView(binding.root)
checkOpenCVLoaded()
getInputImage()
resetImage()
//create example data
createVariableData()
// Set the adapter
binding.rvAppVariables.layoutManager = LinearLayoutManager(this, RecyclerView.VERTICAL, false)
binding.rvAppVariables.adapter = appVariableAdapter
binding.rvAppVariables.addItemDecoration(DividerItemDecoration(this, DividerItemDecoration.VERTICAL))
setupDropDownMenu()
bindButtons()
addButtonListeners()
updateAppVariable(1)
}
private fun checkOpenCVLoaded()
{
if(OpenCVLoader.initDebug())
Toast.makeText(applicationContext, "OpenCV loaded successfully", Toast.LENGTH_SHORT).show()
else
Toast.makeText(applicationContext, "Could not load OpenCV", Toast.LENGTH_SHORT).show()
}
private fun getInputImage()
{
var testDrawingResource = intent.getIntExtra("testDrawingResource", 0)
if(testDrawingResource == 0)
{
Snackbar.make(binding.ivPreviewImage, "Couldn't load the selected test drawing", Snackbar.LENGTH_SHORT).show()
finish()
return
}
rawInputImage = BitmapFactory.decodeResource(resources, testDrawingResource)
}
private fun setupDropDownMenu()
{
operationsDropdown = binding.ddOperations
var dropdownNames = arrayOf("0. Transform Perspective", "1. Grayscale", "2. Gaussian Blur", "3. Canny (Edge Detection)", "4. Morph", "5. Contour", "6. Crop")
var adapter = ArrayAdapter(this, R.layout.support_simple_spinner_dropdown_item, dropdownNames)
operationsDropdown.adapter = adapter
nOperations = dropdownNames.size
operationsDropdown.setOnItemSelectedListener(object : OnItemSelectedListener {
override fun onItemSelected(adapterView: AdapterView<*>?, view: View?, i: Int, l: Long) {
// Your code here
updateSelectedOperation()
updateAppVariable(selectedOperation)
}
override fun onNothingSelected(adapterView: AdapterView<*>?) {
return
}
})
}
private fun bindButtons()
{
resetButton = binding.btnPreview
applyButton = binding.btnApply
undoButton = binding.btnUndo
}
private fun updateAppVariable(index: Int)
{
appVariables.clear()
if(variables[index].isNotEmpty())
appVariables.addAll(variables[index].values)
appVariableAdapter.notifyDataSetChanged()
}
private fun addButtonListeners()
{
resetButton.setOnClickListener {
resetImage()
updateSelectedOperation()
operationsDropdown.setSelection(0)
}
applyButton.setOnClickListener{
displayProcessedImage()
storeProcessedImage()
updateSelectedOperation()
operationsDropdown.setSelection((selectedOperation+1) % nOperations)
}
undoButton.setOnClickListener{
undo()
updateSelectedOperation()
operationsDropdown.setSelection(selectedOperation-1 % nOperations)
}
}
private fun resetImage()
{
images.clear()
mats.clear()
currentImage = Mat()
bitmapToMat(rawInputImage, currentImage)
images.add(rawInputImage)
mats.add(currentImage)
selectedImageIndex = 0
binding.ivPreviewImage.setImageBitmap(rawInputImage)
}
private fun displayProcessedImage()
{
updateSelectedOperation()
when(selectedOperation){
0 -> transformPerspective()
1 -> getGrayscaleImage()
2 -> getGaussianBlurImage(
Size(variables[2]["width"]!!.value.toDouble(), variables[2]["height"]!!.value.toDouble()),
variables[2]["sigmaX"]!!.value.toDouble(),
variables[2]["sigmaY"]!!.value.toDouble())
3 -> getCannyImage(
variables[3]["cannyMinThreshold"]!!.value.toDouble(),
variables[3]["ratio"]!!.value.toInt(),
variables[3]["apertureSize"]!!.value.toInt())
4 -> getMorphImage(
variables[4]["dilateSize"]!!.value.toDouble(),
variables[4]["dilateIterations"]!!.value.toInt(),
variables[4]["erodeIterations"]!!.value.toInt(),
variables[4]["dilatePointX"]!!.value.toDouble(),
variables[4]["dilatePointY"]!!.value.toDouble(),
variables[4]["erodePointX"]!!.value.toDouble(),
variables[4]["erodePointY"]!!.value.toDouble())
5 -> getContourImage()
6 -> getCroppedImage()
else -> Log.d("error", "selectedOperation out of bounds")
}
binding.ivPreviewImage.setImageBitmap(images[images.size - 1])
}
private fun updateSelectedOperation()
{
//get the name of the selected item in the dropdown menu
val dropdownText: String = operationsDropdown.selectedItem.toString()
Log.d("selectedOperation", dropdownText)
//store the number(index) that's in front of the name
selectedOperation = dropdownText[0].toString().toInt()
}
private fun storeProcessedImage()
{
selectedImageIndex++
currentImage = mats[mats.size - 1]
}
private fun undo()
{
if(selectedImageIndex == 0)
{
Snackbar.make(binding.ivPreviewImage, "Can't undo more steps", Snackbar.LENGTH_SHORT).show()
return
}
selectedImageIndex--
mats.removeAt(mats.size - 1)
images.removeAt(images.size - 1)
currentImage = mats[mats.size - 1]
binding.ivPreviewImage.setImageBitmap(images[images.size - 1])
}
private fun createVariableData()
{
storeVariableData(2, AppVariable("width", 0f))
storeVariableData(2, AppVariable("height", 0f))
storeVariableData(2, AppVariable("sigmaX", 11f))
storeVariableData(2, AppVariable("sigmaY", 11f))
storeVariableData(3, AppVariable("cannyMinThreshold", 4.5f)) //Tussen 4 - 8
storeVariableData(3, AppVariable("ratio", 3f)) //recommended 2:1 and 3:1
storeVariableData(3, AppVariable("apertureSize", 3f))
storeVariableData(4, AppVariable("dilateSize", 10f))
storeVariableData(4, AppVariable("dilateIterations", 6f))
storeVariableData(4, AppVariable("erodeIterations", 6f))
storeVariableData(4, AppVariable("dilatePointX", -1f))
storeVariableData(4, AppVariable("dilatePointY", -1f))
storeVariableData(4, AppVariable("erodePointX", -1f))
storeVariableData(4, AppVariable("erodePointY", -1f))
}
private fun storeVariableData(index: Int, variable: AppVariable)
{
variables[index][variable.name] = variable
}
//gets called from the Recyclerview
override fun updateVariable(appVariable: AppVariable)
{
variables[selectedOperation][appVariable.name] = appVariable
}
private fun transformPerspective()
{
val imageHeight = rawInputImage.height
val imageWidth = rawInputImage.width
val mat = Mat()
bitmapToMat(rawInputImage, mat)
cvtColor(mat, mat, COLOR_RGB2RGBA)
val bovekantPct = 39 // percentage vanaf boven wat eraf gaat
val onderkantPct = 87 // percentage vanaf boven wat in beeld komt
val breedtePct = 50 // Hoeveel de bovenkant smaller wordt vergeleken met de onderkant
val linksRechtsPct = 13 // Het percentage wat er links en rechts af gaat
val leftMargin = imageWidth * linksRechtsPct.toDouble() / 100
val srcWidth = imageWidth * (100 - linksRechtsPct.toDouble() * 2) / 100
val linksBovenX = (100-breedtePct).toDouble()/200 * srcWidth + leftMargin
val rechtsBovenX = (imageWidth - linksBovenX)
val bovenY = imageHeight.toDouble() * (bovekantPct.toDouble()/100)
val onderY = imageHeight.toDouble() * (onderkantPct.toDouble()/100)
val linksOnderX = leftMargin
val rechtsOnderX = imageWidth - leftMargin
val srcmat = MatOfPoint2f(
Point(linksBovenX, bovenY),
Point(rechtsBovenX, bovenY),
Point(linksOnderX, onderY),
Point(rechtsOnderX, onderY)
)
//Input image
val dstmat = MatOfPoint2f(
Point(0.0, 0.0),
Point(imageWidth.toDouble() - 1, 0.0),
Point(0.0, imageHeight.toDouble() - 1),
Point(imageWidth.toDouble() - 1, imageHeight.toDouble() - 1)
)
val perspectiveTransform = getPerspectiveTransform(srcmat, dstmat)
val dst = mat.clone()
warpPerspective(mat, dst, perspectiveTransform, Size(imageWidth + 0.0, imageHeight + 0.0))
var outputImage = Bitmap.createBitmap(dst.cols(), dst.rows(), Bitmap.Config.ARGB_8888)
matToBitmap(dst, outputImage)
images.add(outputImage)
mats.add(dst)
}
private fun getGrayscaleImage()
{
val inputMat = mats[mats.size - 1]
cvtColor(inputMat, inputMat, COLOR_RGB2RGBA)
//!grayscale
val grayscaleMat = Mat()
cvtColor(inputMat, grayscaleMat, COLOR_RGB2GRAY)
//show grayscale image
var grayscaleImage = Bitmap.createBitmap(
grayscaleMat.cols(),
grayscaleMat.rows(),
Bitmap.Config.ARGB_8888
)
matToBitmap(grayscaleMat, grayscaleImage)
images.add(grayscaleImage)
mats.add(grayscaleMat)
}
private fun getGaussianBlurImage(point: Size, sigmaX: Double, sigmaY: Double)
{
val inputMat = mats[mats.size - 1]
//!GaussianBlur
val gaussianBlurMat = Mat()
GaussianBlur(inputMat, gaussianBlurMat, point, sigmaX, sigmaY)
//show gaussian blur image
var gaussianImage = Bitmap.createBitmap(
gaussianBlurMat.cols(),
gaussianBlurMat.rows(),
Bitmap.Config.ARGB_8888
)
matToBitmap(gaussianBlurMat, gaussianImage)
images.add(gaussianImage)
mats.add(gaussianBlurMat)
}
private fun getCannyImage(cannyMinThreshold: Double, ratio: Int, apertureSize: Int)
{
val inputMat = mats[mats.size - 1]
//!Canny
val cannyMat = Mat()
inputMat.copyTo(cannyMat)
Canny(cannyMat, cannyMat, cannyMinThreshold, cannyMinThreshold * ratio, apertureSize)
//show canny mask image
var cannyImage = Bitmap.createBitmap(cannyMat.cols(), cannyMat.rows(), Bitmap.Config.ARGB_8888)
matToBitmap(cannyMat, cannyImage)
images.add(cannyImage)
mats.add(cannyMat)
}
private fun getMorphImage(dilateSize: Double, dilateIterations: Int, erodeIterations: Int,
dilatePointX: Double, dilatePointY: Double, erodePointX: Double, erodePointY: Double)
{
val inputMat = mats[mats.size - 1]
//!Morphological transformation
val cannyMorphMat = Mat()
inputMat.copyTo(cannyMorphMat)
val morphKernel = getStructuringElement(MORPH_ELLIPSE, Size(dilateSize, dilateSize))
//groter maken van de lijnen/ vastmaken aan elkaar
dilate(cannyMorphMat, cannyMorphMat, morphKernel, Point(dilatePointX, dilatePointY), dilateIterations)
//"wegvreten"
erode(cannyMorphMat, cannyMorphMat, morphKernel, Point(erodePointX, erodePointY), erodeIterations)
//show morph
var morphImage = Bitmap.createBitmap(
cannyMorphMat.cols(),
cannyMorphMat.rows(),
Bitmap.Config.ARGB_8888
)
matToBitmap(cannyMorphMat, morphImage)
images.add(morphImage)
mats.add(cannyMorphMat)
}
private fun getContourImage()
{
val inputMat = mats[mats.size - 1]
//!Colour in section
var white : Scalar = Scalar(255.0, 255.0, 255.0)
var contours: List<MatOfPoint> = ArrayList()
var hierarchy = Mat()
findContours(inputMat, contours, hierarchy, RETR_TREE, CHAIN_APPROX_SIMPLE)
//filled = -1
//drawContours(cannyMorphMat, contours, -1, white, -1)
//fill contours in with white
for (contour in contours)
fillPoly(inputMat, listOf(contour), white)
//show contours
var contourImage = Bitmap.createBitmap(
inputMat.cols(),
inputMat.rows(),
Bitmap.Config.ARGB_8888
)
matToBitmap(inputMat, contourImage)
images.add(contourImage)
mats.add(inputMat)
}
private fun getCroppedImage()
{
val startingMat = mats[1]
val inputMat = mats[mats.size - 1]
//!cut out input image
var mask : Mat = inputMat
var transparentBackground : Scalar = Scalar(0.0, 0.0, 0.0, 0.0)
var crop = Mat(startingMat.size(), startingMat.type(), transparentBackground)
startingMat.copyTo(crop, mask)
//show output image
var outputImage = Bitmap.createBitmap(crop.cols(), crop.rows(), Bitmap.Config.ARGB_8888)
matToBitmap(crop, outputImage)
images.add(outputImage)
mats.add(crop)
}
}