vgabios: Initial support for fixing up assembler to workaround x86emu.

Perform post-processing of the vgabios assembler to remove certain
instructions that gcc generates and x86emu can't handle.

Signed-off-by: Kevin O'Connor <kevin@koconnor.net>

1 files changed, 39 insertions, 0 deletions

diff --git a/tools/vgafixup.py b/tools/vgafixup.py
new file mode 100644
index 0000000..68e5d1f
--- /dev/null
+++ b/tools/vgafixup.py
@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+# Work around x86emu bugs by replacing problematic instructions.
+#
+# Copyright (C) 2012  Kevin O'Connor <kevin@koconnor.net>
+#
+# This file may be distributed under the terms of the GNU GPLv3 license.
+
+# The x86emu code widely used in Linux distributions when running Xorg
+# in vesamode is known to have issues with "retl", "leavel", "entryl",
+# and some variants of "calll".  This code modifies those instructions
+# (ret and leave) that are known to be generated by gcc to avoid
+# triggering the x86emu bugs.
+
+# It is also known that the Windows vgabios emulator has issues with
+# addressing negative offsets to the %esp register.  That has been
+# worked around by not using the gcc paremeter "-fomit-frame-pointer"
+# when compiling.
+
+import sys
+
+def main():
+    infilename, outfilename = sys.argv[1:]
+    infile = open(infilename, 'rb')
+    out = []
+    for line in infile:
+        sline = line.strip()
+        if sline == 'ret':
+            out.append('retw $2\n')
+        elif sline == 'leave':
+            out.append('movl %ebp, %esp ; popl %ebp\n')
+        else:
+            out.append(line)
+    infile.close()
+    outfile = open(outfilename, 'wb')
+    outfile.write(''.join(out))
+    outfile.close()
+
+if __name__ == '__main__':
+    main()