Merge branch 'master' into master

CHANGELOG.md

@@ -1,5 +1,8 @@
 |Version|Date|Notes|
 |---|---|---|
+|4.3|2021-03-24|Enable interactive QEMU again. Remove envsubst since we are already using bash... Add set -x flag| 
+|4.2|2021-03-24|Add all ENV variables to each dockerfile for readability. Add RAM allocation buffer and cache drop bug fix. Add kvm and libvirt groups. Add `IMAGE_FORMAT=qcow2` to allow `IMAGE_FORMAT=raw` too.|
+|   |2021-03-19|Use RAM=3 as the default RAM allocation. Add instructions to clear buff/cache.|
 |   |2021-03-17|Add RAM=max and RAM=half to dynamically select ram at runtime (DEFAULT).|
 |   |2021-03-06|Change envs to require --envs. Automatically enable --envs if --output-env is used. Same for plists, bootdisks. Fix help ugliness and sanity of generate serial scripts. Fix bootdisk not getting written to persistent file when using NOPICKER=true. NOPICKER=true is overridden by a custom plist now anyway. Remove useless case statements. Allow -e HEADLESS=true as human readable alternative to -e DISPLAY=:99.|
 |4.1|2021-03-04|Add `-e MASTER_PLIST_URL` to all images to allow using your own remote plist.|

CREDITS.md

@@ -1,8 +1,12 @@
 # Credits
 
-The upstream credits are available at [@Kholia](https://github.com/Kholia)'s repo https://github.com/kholia/OSX-KVM/blob/master/CREDITS.md
+## Upstream Acknowledgements:
 
-These credits refer to the contributors to this repository:
+This project uses OSX-KVM from https://github.com/kholia/OSX-KVM/ and fully appreciates the work done by [@Kholia](https://github.com/Kholia) and all the contributors who are listed: [https://github.com/kholia/OSX-KVM/blob/master/CREDITS.md](https://github.com/kholia/OSX-KVM/blob/master/CREDITS.md)
+
+This project now uses the fantastic OpenCore bootloader from the community OpenCore project: https://github.com/acidanthera/OpenCorePkg. You can join their [Subreddit here](https://www.reddit.com/r/hackintosh/)!
+
+## These credits refer to the contributors to this repository:
 
 [@GreeFine](https://github.com/GreeFine) - Readme Typo #9 
 

Dockerfile

@@ -7,7 +7,7 @@
 #
 # Title:            Docker-OSX (Mac on Docker)
 # Author:           Sick.Codes https://twitter.com/sickcodes
-# Version:          4.1
+# Version:          4.3
 # License:          GPLv3+
 # Repository:       https://github.com/sickcodes/Docker-OSX
 # Website:          https://sick.codes
@@ -165,6 +165,8 @@ RUN patched_glibc=glibc-linux4-2.33-4-x86_64.pkg.tar.zst \
 
 WORKDIR /home/arch/OSX-KVM
 
+RUN wget https://raw.githubusercontent.com/sickcodes/Docker-OSX/master/fetch-macOS.py
+
 RUN [[ "${VERSION%%.*}" -lt 11 ]] && { python fetch-macOS.py --version "${VERSION}" \
         && qemu-img convert BaseSystem.dmg -O qcow2 -p -c BaseSystem.img \
         && qemu-img create -f qcow2 mac_hdd_ng.img "${SIZE}" \
@@ -203,19 +205,10 @@ ARG BRANCH=master
 ARG REPO='https://github.com/sickcodes/Docker-OSX.git'
 RUN git clone --recurse-submodules --depth 1 --branch "${BRANCH}" "${REPO}"
 
-# env -e ADDITIONAL_PORTS with a comma
-# for example, -e ADDITIONAL_PORTS=hostfwd=tcp::23-:23,
-ENV ADDITIONAL_PORTS=
-
-# dynamic RAM options for runtime
-ENV RAM=3
-# ENV RAM=max
-# ENV RAM=half
-
 RUN touch Launch.sh \
     && chmod +x ./Launch.sh \
-    && tee -a Launch.sh <<< '#!/bin/sh' \
+    && tee -a Launch.sh <<< '#!/bin/bash' \
-    && tee -a Launch.sh <<< 'set -eu' \
+    && tee -a Launch.sh <<< 'set -eux' \
     && tee -a Launch.sh <<< 'sudo chown    $(id -u):$(id -g) /dev/kvm 2>/dev/null || true' \
     && tee -a Launch.sh <<< 'sudo chown -R $(id -u):$(id -g) /dev/snd 2>/dev/null || true' \
     && tee -a Launch.sh <<< '[[ "${RAM}" = max ]] && export RAM="$(("$(head -n1 /proc/meminfo | tr -dc "[:digit:]") / 1000000"))"' \
@@ -223,7 +216,7 @@ RUN touch Launch.sh \
     && tee -a Launch.sh <<< 'sudo chown -R $(id -u):$(id -g) /dev/snd 2>/dev/null || true' \
     && tee -a Launch.sh <<< 'exec qemu-system-x86_64 -m ${RAM:-2}000 \' \
     && tee -a Launch.sh <<< '-cpu Penryn,vendor=GenuineIntel,+invtsc,vmware-cpuid-freq=on,+pcid,+ssse3,+sse4.2,+popcnt,+avx,+aes,+xsave,+xsaveopt,check \' \
-    && tee -a Launch.sh <<< '-machine q35,accel=kvm:tcg \' \
+    && tee -a Launch.sh <<< '-machine q35,${KVM-"accel=kvm:tcg"} \' \
     && tee -a Launch.sh <<< '-smp ${CPU_STRING:-${SMP:-4},cores=${CORES:-4}} \' \
     && tee -a Launch.sh <<< '-usb -device usb-kbd -device usb-tablet \' \
     && tee -a Launch.sh <<< '-device isa-applesmc,osk=ourhardworkbythesewordsguardedpleasedontsteal\(c\)AppleComputerInc \' \
@@ -236,7 +229,7 @@ RUN touch Launch.sh \
     && tee -a Launch.sh <<< '-device ide-hd,bus=sata.2,drive=OpenCoreBoot \' \
     && tee -a Launch.sh <<< '-device ide-hd,bus=sata.3,drive=InstallMedia \' \
     && tee -a Launch.sh <<< '-drive id=InstallMedia,if=none,file=/home/arch/OSX-KVM/BaseSystem.img,format=qcow2 \' \
-    && tee -a Launch.sh <<< '-drive id=MacHDD,if=none,file=${IMAGE_PATH:-/home/arch/OSX-KVM/mac_hdd_ng.img},format=qcow2 \' \
+    && tee -a Launch.sh <<< '-drive id=MacHDD,if=none,file=${IMAGE_PATH:-/home/arch/OSX-KVM/mac_hdd_ng.img},format=${IMAGE_FORMAT:-qcow2} \' \
     && tee -a Launch.sh <<< '-device ide-hd,bus=sata.4,drive=MacHDD \' \
     && tee -a Launch.sh <<< '-netdev user,id=net0,hostfwd=tcp::${INTERNAL_SSH_PORT:-10022}-:22,hostfwd=tcp::${SCREEN_SHARE_PORT:-5900}-:5900,${ADDITIONAL_PORTS} \' \
     && tee -a Launch.sh <<< '-device ${NETWORKING:-vmxnet3},netdev=net0,id=net0,mac=${MAC_ADDRESS:-52:54:00:09:49:17} \' \
@@ -255,32 +248,47 @@ USER arch
 
 ENV USER arch
 
+#### SPECIAL RUNTIME ARGUMENTS BELOW
+
+# env -e ADDITIONAL_PORTS with a comma
+# for example, -e ADDITIONAL_PORTS=hostfwd=tcp::23-:23,
+ENV ADDITIONAL_PORTS=
+
 ENV BOOTDISK=
 
 ENV DISPLAY=:0.0
 
 ENV ENV=/env
 
-ENV IMAGE_PATH=/home/arch/OSX-KVM/mac_hdd_ng.img
-
-# ENV NETWORKING=e1000-82545em
-ENV NETWORKING=vmxnet3
-
 # Boolean for generating a bootdisk with new random serials.
 ENV GENERATE_UNIQUE=false
 
 # Boolean for generating a bootdisk with specific serials.
 ENV GENERATE_SPECIFIC=false
 
+ENV IMAGE_PATH=/home/arch/OSX-KVM/mac_hdd_ng.img
+ENV IMAGE_FORMAT=qcow2
+
+ENV KVM='accel=kvm:tcg'
+
+ENV MASTER_PLIST_URL="https://raw.githubusercontent.com/sickcodes/osx-serial-generator/master/config-nopicker-custom.plist"
+
+# ENV NETWORKING=e1000-82545em
+ENV NETWORKING=vmxnet3
+
 # boolean for skipping the disk selection menu at in the boot process
 ENV NOPICKER=false
 
+# dynamic RAM options for runtime
+ENV RAM=3
+# ENV RAM=max
+# ENV RAM=half
+
 # The x and y coordinates for resolution.
 # Must be used with either -e GENERATE_UNIQUE=true or -e GENERATE_SPECIFIC=true.
 ENV WIDTH=1920
 ENV HEIGHT=1080
 
-ENV MASTER_PLIST_URL="https://raw.githubusercontent.com/sickcodes/osx-serial-generator/master/config-nopicker-custom.plist"
 
 VOLUME ["/tmp/.X11-unix"]
 
@@ -334,7 +342,7 @@ CMD sudo touch /dev/kvm /dev/snd "${IMAGE_PATH}" "${BOOTDISK}" "${ENV}" || true
             --height "${HEIGHT:-1080}" \
             --output-bootdisk "${BOOTDISK:=/home/arch/OSX-KVM/OpenCore-Catalina/OpenCore.qcow2}" \
     ; } \
-    ; ./enable-ssh.sh && envsubst < ./Launch.sh | bash
+    ; ./enable-ssh.sh && /bin/bash -c ./Launch.sh
 
 # virt-manager mode: eta son
 # CMD virsh define <(envsubst < Docker-OSX.xml) && virt-manager || virt-manager

Dockerfile.auto

@@ -7,7 +7,7 @@
 # 
 # Title:            Docker-OSX (Mac on Docker)
 # Author:           Sick.Codes https://twitter.com/sickcodes
-# Version:          4.1
+# Version:          4.3
 # License:          GPLv3+
 # Repository:       https://github.com/sickcodes/Docker-OSX
 # Website:          https://sick.codes
@@ -115,20 +115,25 @@ RUN mkdir -p ~/.ssh \
 
 ARG COMPLETE=true
 
-# Feel free to take a copy of this image and then host it internally
-ARG IMAGE_URL='https://images2.sick.codes/mac_hdd_ng_auto.img'
-
 # use the COMPLETE arg, for a complete image, ready to boot.
 # otherwise use your own image: -v "$PWD/disk.img":/image
 ARG WGET_OPTIONS=
 # ARG WGET_OPTIONS='--no-verbose'
 
+# Feel free to take a copy of this image and then host it internally
+ARG IMAGE_URL='https://images.sick.codes/mac_hdd_ng_auto.img'
+# ARG IMAGE_URL='https://images.sick.codes/mac_hdd_ng_auto_big_sur.img'
+
 RUN if [[ "${COMPLETE}" ]]; then \
-        echo "Downloading 20GB+ image... This step might take a while... Press Ctrl+C if you want to abort." \
+        echo "Downloading 20GB image... This step might take a while... Press Ctrl+C if you want to abort." \
         ; rm -f /home/arch/OSX-KVM/mac_hdd_ng.img \
         && wget ${WGET_OPTIONS} -O /home/arch/OSX-KVM/mac_hdd_ng.img "${IMAGE_URL}" \
     ; fi
 
+#### SPECIAL RUNTIME ARGUMENTS BELOW
+
+ENV ADDITIONAL_PORTS=
+
 ENV BOOTDISK=
 
 ENV DISPLAY=:99
@@ -137,10 +142,34 @@ ENV HEADLESS=false
 
 ENV ENV=/env
 
+# Boolean for generating a bootdisk with new random serials.
+ENV GENERATE_UNIQUE=false
+
+# Boolean for generating a bootdisk with specific serials.
+ENV GENERATE_SPECIFIC=false
+
 ENV IMAGE_PATH=/home/arch/OSX-KVM/mac_hdd_ng.img
+ENV IMAGE_FORMAT=qcow2
+
+ENV KVM='accel=kvm:tcg'
+
+# ENV MASTER_PLIST_URL="https://raw.githubusercontent.com/sickcodes/osx-serial-generator/master/config-custom.plist"
+
+# ENV NETWORKING=e1000-82545em
+ENV NETWORKING=vmxnet3
 
 ENV NOPICKER=true
 
+# dynamic RAM options for runtime
+ENV RAM=3
+# ENV RAM=max
+# ENV RAM=half
+
+# The x and y coordinates for resolution.
+# Must be used with either -e GENERATE_UNIQUE=true or -e GENERATE_SPECIFIC=true.
+ENV WIDTH=1920
+ENV HEIGHT=1080
+
 ENV TERMS_OF_USE=i_agree
 
 ENV BOILERPLATE="By using this Dockerfile, you hereby agree that you are a security reseacher or developer and agree to use this Dockerfile to make the world a safer place. Examples include: making your apps safer, finding your mobile phone, compiling security products, etc. You understand that Docker-OSX is an Open Source project, which is released to the public under the GNU Pulic License version 3 and above. You acknowledge that the Open Source project is absolutely unaffiliated with any third party, in any form whatsoever. Any trademarks or intelectual property which happen to be mentioned anywhere in or around the project are owned by their respective owners. By using this Dockerfile, you agree to agree to the EULA of each piece of upstream or downstream software. The following code is released for the sole purpose of security research, under the GNU Public License version 3. If you are concerned about the licensing, please note that this project is not AGPL. A copy of the license is available online: https://github.com/sickcodes/Docker-OSX/blob/master/LICENSE. In order to use the following Dockerfile you must read and understand the terms. Once you have read the terms, use the -e TERMS_OF_USE=i_agree or -e TERMS_OF_USE=i_disagree"

Dockerfile.naked

@@ -7,7 +7,7 @@
 # 
 # Title:            Docker-OSX (Mac on Docker)
 # Author:           Sick.Codes https://twitter.com/sickcodes
-# Version:          4.1
+# Version:          4.3
 # License:          GPLv3+
 # Repository:       https://github.com/sickcodes/Docker-OSX
 # Website:          https://sick.codes
@@ -102,6 +102,10 @@ RUN mkdir -p ~/.ssh \
     && tee -a ~/.ssh/config <<< '    StrictHostKeyChecking no' \
     && tee -a ~/.ssh/config <<< '    UserKnownHostsFile=/dev/null'
 
+#### SPECIAL RUNTIME ARGUMENTS BELOW
+
+ENV ADDITIONAL_PORTS=
+
 ENV BOOTDISK=
 
 ENV DISPLAY=:99
@@ -110,10 +114,34 @@ ENV HEADLESS=false
 
 ENV ENV=/env
 
+# Boolean for generating a bootdisk with new random serials.
+ENV GENERATE_UNIQUE=false
+
+# Boolean for generating a bootdisk with specific serials.
+ENV GENERATE_SPECIFIC=false
+
 ENV IMAGE_PATH=/image
+ENV IMAGE_FORMAT=qcow2
+
+ENV KVM='accel=kvm:tcg'
+
+# ENV MASTER_PLIST_URL="https://raw.githubusercontent.com/sickcodes/osx-serial-generator/master/config-custom.plist"
+
+# ENV NETWORKING=e1000-82545em
+ENV NETWORKING=vmxnet3
 
 ENV NOPICKER=true
 
+# dynamic RAM options for runtime
+ENV RAM=3
+# ENV RAM=max
+# ENV RAM=half
+
+# The x and y coordinates for resolution.
+# Must be used with either -e GENERATE_UNIQUE=true or -e GENERATE_SPECIFIC=true.
+ENV WIDTH=1920
+ENV HEIGHT=1080
+
 CMD sudo touch /dev/kvm /dev/snd "${IMAGE_PATH}" "${BOOTDISK}" "${ENV}" || true \
     ; sudo chown -R $(id -u):$(id -g) /dev/kvm /dev/snd "${IMAGE_PATH}" "${BOOTDISK}" "${ENV}" || true \
     ; { [[ "${DISPLAY}" = ':99' ]] || [[ "${HEADLESS}" == true ]] ; } && { \

README.md

@@ -42,6 +42,10 @@ The images (excluding `:naked`) launch a container with an existing installation
 - gpu acceleration
 - support for virt-manager
 
+Big thanks to the OpenCore team over at: https://github.com/acidanthera/OpenCorePkg. Their well-maintained bootloader provides much of the great functionality that Docker-OSX users enjoy :)
+
+If you like this project, consider contributing upstream!
+
 ## Docker
 
 Images built on top of the contents of this repository are also available on **Docker Hub** for convenience: https://hub.docker.com/r/sickcodes/docker-osx
@@ -84,7 +88,7 @@ In case you're interested, contact [@sickcodes on Twitter](https://twitter.com/s
 
 ## License/Contributing
 
-Docker-OSX is licensed under the [GPL v3](LICENSE). Contributions are welcomed and immensely appreciated.
+Docker-OSX is licensed under the [GPL v3+](LICENSE). Contributions are welcomed and immensely appreciated. You are in-fact permitted to use Docker-OSX as a tool to create proprietary software.
 
 ### Other cool Docker/QEMU based projects
 
@@ -92,7 +96,9 @@ Docker-OSX is licensed under the [GPL v3](LICENSE). Contributions are welcomed a
 
 ## Disclaimer
 
-Product names, logos, brands and other trademarks referred to within this project are the property of their respective trademark holders. These trademark holders are not affiliated with our repository in any capacity. They do not sponsor or endorse our materials.
+If you are serious about Apple Security, and possibly finding 6-figure bug bounties within the Apple Bug Bounty Program, then you're in the right place! Further notes: [Is Hackintosh, OSX-KVM, or Docker-OSX legal?](https://sick.codes/is-hackintosh-osx-kvm-or-docker-osx-legal/).
+
+Product names, logos, brands and other trademarks referred to within this project are the property of their respective trademark holders. These trademark holders are not affiliated with our repository in any capacity. They do not sponsor or endorse this project in any way.
 
 ## Instructions
 
@@ -109,6 +115,7 @@ docker run -it \
     -e "DISPLAY=${DISPLAY:-:0.0}" \
     sickcodes/docker-osx:latest
 
+docker pull sickcodes/docker-osx:big-sur
 # Big Sur
 docker run -it \
     --device /dev/kvm \
@@ -137,7 +144,7 @@ Create your personal image using `:latest`. Then, extract the image. Afterwards,
 
 The Quick Start command should work out of the box, provided that you keep the following lines. Works in `auto` & `naked` machines:
 
-```dockerfile
+```
     -v /tmp/.X11-unix:/tmp/.X11-unix \
     -e "DISPLAY=${DISPLAY:-:0.0}" \
 ```
@@ -146,11 +153,48 @@ The Quick Start command should work out of the box, provided that you keep the f
 
 In that case, **remove** the two lines in your command:
 
-```dockerfile
+```
     # -v /tmp/.X11-unix:/tmp/.X11-unix \
     # -e "DISPLAY=${DISPLAY:-:0.0}" \
 ```
 
+#### I need VNC to a Remote Host (Secure)
+
+Now you can direct connect VNC to any image!
+
+Add the following line:
+
+`-e EXTRA="-display none -vnc 0.0.0.0:99,password"`
+
+In the Docker terminal, press `enter` until you see `(qemu)`.
+
+Type `change vnc password`
+
+`ip n` will usually show the container IP first.
+
+Port is `5999`.
+
+Now VNC connect using the Docker container IP, for example `172.17.0.2:5999`
+
+You can also find the container IP: `docker inspect <containerid> | jq -r '.[0].NetworkSettings.IPAddress'`
+
+Remote VNC over SSH: `ssh -N root@1.1.1.1 -L  5999:172.17.0.2:5999`, where `1.1.1.1` is your remote server IP and `172.17.0.2` is your LAN container IP.
+
+#### I need VNC on localhost (Local use only!)
+
+##### VNC Insecure
+
+**NOT TLS/HTTPS Encrypted at all!**
+```
+-p 5999:5999
+-e EXTRA="-display none -vnc 0.0.0.0:99,password"
+```
+VNC Connect to `localhost:5999`.
+
+Or `ssh -N root@1.1.1.1 -L  5999:127.0.0.1:5999`, where `1.1.1.1` is your remote server IP.
+
+(Note: if you close port 5999 and use the SSH tunnel, this becomes secure.)
+
 #### I have used Docker-OSX before and wish to extract my Mac OS X image.
 
 Use `docker commit`, copy the ID, and then run `docker start -ai <Replace this with your ID>`.
@@ -421,9 +465,13 @@ docker run \
 
 ## Troubleshooting
 
+Big thank you to our contributors who have worked out almost every conceivable issue so far!
+
 ### LibGTK - Permission denied
 
-Thanks [@raoulh](https://github.com/raoulh) and [@arsham](https://github.com/arsham) for contributing this section.
+[https://github.com/sickcodes/Docker-OSX/blob/master/CREDITS.md](https://github.com/sickcodes/Docker-OSX/blob/master/CREDITS.md)
+
+#### libgtk permissions denied error
 
 ```bash
 echo $DISPLAY
@@ -441,6 +489,38 @@ sudo yum install xorg-x11-server-utils
 xhost +
 
 ```
+#### RAM over-allocation Error
+Cause by trying to allocate more ram to the container than you currently have available for allocation: `cannot set up guest memory 'pc.ram': Cannot allocate memory`.
+
+For example:
+
+```console
+[user@hostname ~]$ free -mh
+               total        used        free      shared  buff/cache   available
+Mem:            30Gi       3.5Gi       7.0Gi       728Mi        20Gi        26Gi
+Swap:           11Gi          0B        11Gi
+```
+
+In the example above, the `buff/cache` already contains 20 Gigabytes of allocated RAM.
+
+Clear the buffer and the cache:
+
+```bash
+sudo tee /proc/sys/vm/drop_caches <<< 3
+```
+
+Now check the ram again:
+
+```console
+[user@hostname ~]$ free -mh
+               total        used        free      shared  buff/cache   available
+Mem:            30Gi       3.3Gi        26Gi       697Mi       1.5Gi        26Gi
+Swap:           11Gi          0B        11Gi
+```
+
+Of course you cannot allocate more RAM that your have. The default is 3 Gigabytes: `-e RAM=3`.
+
+#### PulseAudio
 
 ### Use PulseAudio for sound 
 
@@ -467,34 +547,35 @@ docker run \
     sickcodes/docker-osx pactl list
 ```
 
-#### Alternative soltuion
+#### Nested Hardware Virtualization
 
-Thanks [@roryrjb](https://github.com/roryrjb) for contributing this section.
+Check if your PC has hardware virtualization enabled:
 
 ```bash
-docker run \
+sudo tee /sys/module/kvm/parameters/ignore_msrs <<< 1
-    --privileged \
+
-    --net host \
+egrep -c '(svm|vmx)' /proc/cpuinfo
-    --cap-add=ALL \
-    -v /tmp/.X11-unix:/tmp/.X11-unix \
-    -v /dev:/dev \
-    -v /lib/modules:/lib/modules \
-    sickcodes/docker-osx
 ```
 
 ### Routine checks
 
 #### Confirm that your CPU supports virtualization
 
-```bash
+#### Add yourself to the Docker group, KVM group, libvirt group.
-egrep -c '(svm|vmx)' /proc/cpuinfo
+
-```
+If you use `sudo dockerd` or dockerd is controlled by systemd/systemctl, then you must be in the Docker group:
 
 #### Try adding yourself to the docker group
 
 ```bash
 sudo usermod -aG docker "${USER}"
 ```
+and also to the kvm and libvirt groups:
+
+```bash
+sudo usermod -aG libvirt "${USER}"
+sudo usermod -aG kvm "${USER}"
+```
 
 #### Enable docker daemon
 

fetch-macOS.py

@@ -0,0 +1,447 @@
+#!/usr/bin/env python3
+# encoding: utf-8
+#
+# https://github.com/munki/macadmin-scripts/blob/master/installinstallmacos.py
+#
+# Copyright 2017 Greg Neagle.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Thanks to Tim Sutton for ideas, suggestions, and sample code.
+#
+# Updated in May of 2019 by Dhiru Kholia.
+
+'''installinstallmacos.py
+A tool to download the parts for an Install macOS app from Apple's
+softwareupdate servers and install a functioning Install macOS app onto an
+empty disk image'''
+
+# https://github.com/foxlet/macOS-Simple-KVM/blob/master/tools/FetchMacOS/fetch-macos.py
+# is pretty similar.
+
+
+# Bad hack
+import warnings
+
+warnings.filterwarnings("ignore", category=DeprecationWarning)
+
+import os
+import gzip
+import argparse
+import plistlib
+import subprocess
+
+from xml.dom import minidom
+from xml.parsers.expat import ExpatError
+
+
+import sys
+
+if sys.version_info[0] < 3:
+    import urlparse as urlstuff
+else:
+    import urllib.parse as urlstuff
+# Quick fix for python 3.9 and above
+if sys.version_info[0] == 3 and sys.version_info[1] >= 9:
+    from types import MethodType
+
+    def readPlist(self,filepath):
+        with open(filepath, 'rb') as f:
+            p = plistlib._PlistParser(dict)
+            rootObject = p.parse(f)
+        return rootObject
+    # adding the method readPlist() to plistlib
+    plistlib.readPlist = MethodType(readPlist, plistlib)
+
+# https://github.com/foxlet/macOS-Simple-KVM/blob/master/tools/FetchMacOS/fetch-macos.py (unused)
+# https://github.com/munki/macadmin-scripts
+catalogs = {
+    "CustomerSeed": "https://swscan.apple.com/content/catalogs/others/index-10.16customerseed-10.16-10.15-10.14-10.13-10.12-10.11-10.10-10.9-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog",
+    "DeveloperSeed": "https://swscan.apple.com/content/catalogs/others/index-10.16seed-10.16-10.15-10.14-10.13-10.12-10.11-10.10-10.9-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog",
+    "PublicSeed": "https://swscan.apple.com/content/catalogs/others/index-10.16beta-10.16-10.15-10.14-10.13-10.12-10.11-10.10-10.9-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog",
+    "PublicRelease": "https://swscan.apple.com/content/catalogs/others/index-10.16-10.15-10.14-10.13-10.12-10.11-10.10-10.9-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog",
+    "20": "https://swscan.apple.com/content/catalogs/others/index-11-10.15-10.14-10.13-10.12-10.11-10.10-10.9-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog"
+}
+
+
+def get_default_catalog():
+    '''Returns the default softwareupdate catalog for the current OS'''
+    return catalogs["20"]
+    # return catalogs["PublicRelease"]
+    # return catalogs["DeveloperSeed"]
+
+
+class ReplicationError(Exception):
+    '''A custom error when replication fails'''
+    pass
+
+
+def cmd_exists(cmd):
+    return subprocess.call("type " + cmd, shell=True,
+                           stdout=subprocess.PIPE, stderr=subprocess.PIPE) == 0
+
+
+def replicate_url(full_url,
+                  root_dir='/tmp',
+                  show_progress=False,
+                  ignore_cache=False,
+                  attempt_resume=False, installer=False, product_title=""):
+    '''Downloads a URL and stores it in the same relative path on our
+    filesystem. Returns a path to the replicated file.'''
+
+    # hack
+    print("[+] Fetching %s" % full_url)
+    if installer and "BaseSystem.dmg" not in full_url and "Big Sur" not in product_title:
+        return
+    if "Big Sur" in product_title and "InstallAssistant.pkg" not in full_url:
+        return
+    attempt_resume = True
+    # path = urllib.parse.urlsplit(full_url)[2]
+    path = urlstuff.urlsplit(full_url)[2]
+    relative_url = path.lstrip('/')
+    relative_url = os.path.normpath(relative_url)
+    # local_file_path = os.path.join(root_dir, relative_url)
+    local_file_path = relative_url
+    # print("Downloading %s..." % full_url)
+
+    if cmd_exists('wget'):
+        if not installer:
+            download_cmd = ['wget', "-c", "--quiet", "-x", "-nH", full_url]
+            # this doesn't work as there are multiple metadata files with the same name!
+            # download_cmd = ['wget', "-c", "--quiet", full_url]
+        else:
+            download_cmd = ['wget', "-c", full_url]
+    else:
+        if not installer:
+            download_cmd = ['curl', "--silent", "--show-error", "-o", local_file_path, "--create-dirs", full_url]
+        else:
+            local_file_path = os.path.basename(local_file_path)
+            download_cmd = ['curl', "-o", local_file_path, full_url]
+
+    try:
+        subprocess.check_call(download_cmd)
+    except subprocess.CalledProcessError as err:
+        raise ReplicationError(err)
+    return local_file_path
+
+
+def parse_server_metadata(filename):
+    '''Parses a softwareupdate server metadata file, looking for information
+    of interest.
+    Returns a dictionary containing title, version, and description.'''
+    title = ''
+    vers = ''
+    try:
+        md_plist = plistlib.readPlist(filename)
+    except (OSError, IOError, ExpatError) as err:
+        print('Error reading %s: %s' % (filename, err), file=sys.stderr)
+        return {}
+    vers = md_plist.get('CFBundleShortVersionString', '')
+    localization = md_plist.get('localization', {})
+    preferred_localization = (localization.get('English') or
+                              localization.get('en'))
+    if preferred_localization:
+        title = preferred_localization.get('title', '')
+
+    metadata = {}
+    metadata['title'] = title
+    metadata['version'] = vers
+
+    """
+    {'title': 'macOS Mojave', 'version': '10.14.5'}
+    {'title': 'macOS Mojave', 'version': '10.14.6'}
+    """
+    return metadata
+
+
+def get_server_metadata(catalog, product_key, workdir, ignore_cache=False):
+    '''Replicate ServerMetaData'''
+    try:
+        url = catalog['Products'][product_key]['ServerMetadataURL']
+        try:
+            smd_path = replicate_url(
+                url, root_dir=workdir, ignore_cache=ignore_cache)
+            return smd_path
+        except ReplicationError as err:
+            print('Could not replicate %s: %s' % (url, err), file=sys.stderr)
+            return None
+    except KeyError:
+        # print('Malformed catalog.', file=sys.stderr)
+        return None
+
+
+def parse_dist(filename):
+    '''Parses a softwareupdate dist file, returning a dict of info of
+    interest'''
+    dist_info = {}
+    try:
+        dom = minidom.parse(filename)
+    except ExpatError:
+        print('Invalid XML in %s' % filename, file=sys.stderr)
+        return dist_info
+    except IOError as err:
+        print('Error reading %s: %s' % (filename, err), file=sys.stderr)
+        return dist_info
+
+    titles = dom.getElementsByTagName('title')
+    if titles:
+        dist_info['title_from_dist'] = titles[0].firstChild.wholeText
+
+    auxinfos = dom.getElementsByTagName('auxinfo')
+    if not auxinfos:
+        return dist_info
+    auxinfo = auxinfos[0]
+    key = None
+    value = None
+    children = auxinfo.childNodes
+    # handle the possibility that keys from auxinfo may be nested
+    # within a 'dict' element
+    dict_nodes = [n for n in auxinfo.childNodes
+                  if n.nodeType == n.ELEMENT_NODE and
+                  n.tagName == 'dict']
+    if dict_nodes:
+        children = dict_nodes[0].childNodes
+    for node in children:
+        if node.nodeType == node.ELEMENT_NODE and node.tagName == 'key':
+            key = node.firstChild.wholeText
+        if node.nodeType == node.ELEMENT_NODE and node.tagName == 'string':
+            value = node.firstChild.wholeText
+        if key and value:
+            dist_info[key] = value
+            key = None
+            value = None
+    return dist_info
+
+
+def download_and_parse_sucatalog(sucatalog, workdir, ignore_cache=False):
+    '''Downloads and returns a parsed softwareupdate catalog'''
+    try:
+        localcatalogpath = replicate_url(
+            sucatalog, root_dir=workdir, ignore_cache=ignore_cache)
+    except ReplicationError as err:
+        print('Could not replicate %s: %s' % (sucatalog, err), file=sys.stderr)
+        exit(-1)
+    if os.path.splitext(localcatalogpath)[1] == '.gz':
+        with gzip.open(localcatalogpath) as the_file:
+            content = the_file.read()
+            try:
+                catalog = plistlib.readPlistFromString(content)
+                return catalog
+            except ExpatError as err:
+                print('Error reading %s: %s' % (localcatalogpath, err), file=sys.stderr)
+                exit(-1)
+    else:
+        try:
+            catalog = plistlib.readPlist(localcatalogpath)
+            return catalog
+        except (OSError, IOError, ExpatError) as err:
+            print('Error reading %s: %s' % (localcatalogpath, err), file=sys.stderr)
+            exit(-1)
+
+
+def find_mac_os_installers(catalog):
+    '''Return a list of product identifiers for what appear to be macOS
+    installers'''
+    mac_os_installer_products = []
+    if 'Products' in catalog:
+        for product_key in catalog['Products'].keys():
+            product = catalog['Products'][product_key]
+            try:
+                if product['ExtendedMetaInfo'][
+                        'InstallAssistantPackageIdentifiers']:
+                    mac_os_installer_products.append(product_key)
+            except KeyError:
+                continue
+
+    return mac_os_installer_products
+
+
+def os_installer_product_info(catalog, workdir, ignore_cache=False):
+    '''Returns a dict of info about products that look like macOS installers'''
+    product_info = {}
+    installer_products = find_mac_os_installers(catalog)
+    for product_key in installer_products:
+        product_info[product_key] = {}
+        filename = get_server_metadata(catalog, product_key, workdir)
+        if filename:
+            product_info[product_key] = parse_server_metadata(filename)
+        else:
+            # print('No server metadata for %s' % product_key)
+            product_info[product_key]['title'] = None
+            product_info[product_key]['version'] = None
+
+        product = catalog['Products'][product_key]
+        product_info[product_key]['PostDate'] = product['PostDate']
+        distributions = product['Distributions']
+        dist_url = distributions.get('English') or distributions.get('en')
+        try:
+            dist_path = replicate_url(
+                dist_url, root_dir=workdir, ignore_cache=ignore_cache)
+        except ReplicationError as err:
+            print('Could not replicate %s: %s' % (dist_url, err),
+                  file=sys.stderr)
+        else:
+            dist_info = parse_dist(dist_path)
+            product_info[product_key]['DistributionPath'] = dist_path
+            product_info[product_key].update(dist_info)
+            if not product_info[product_key]['title']:
+                product_info[product_key]['title'] = dist_info.get('title_from_dist')
+            if not product_info[product_key]['version']:
+                product_info[product_key]['version'] = dist_info.get('VERSION')
+
+    return product_info
+
+
+def replicate_product(catalog, product_id, workdir, ignore_cache=False, product_title=""):
+    '''Downloads all the packages for a product'''
+    product = catalog['Products'][product_id]
+    for package in product.get('Packages', []):
+        # TO-DO: Check 'Size' attribute and make sure
+        # we have enough space on the target
+        # filesystem before attempting to download
+        if 'URL' in package:
+            try:
+                replicate_url(
+                    package['URL'], root_dir=workdir,
+                    show_progress=True, ignore_cache=ignore_cache,
+                    attempt_resume=(not ignore_cache), installer=True, product_title=product_title)
+            except ReplicationError as err:
+                print('Could not replicate %s: %s' % (package['URL'], err), file=sys.stderr)
+                exit(-1)
+        if 'MetadataURL' in package:
+            try:
+                replicate_url(package['MetadataURL'], root_dir=workdir,
+                              ignore_cache=ignore_cache, installer=True)
+            except ReplicationError as err:
+                print('Could not replicate %s: %s' % (package['MetadataURL'], err), file=sys.stderr)
+                exit(-1)
+
+
+def find_installer_app(mountpoint):
+    '''Returns the path to the Install macOS app on the mountpoint'''
+    applications_dir = os.path.join(mountpoint, 'Applications')
+    for item in os.listdir(applications_dir):
+        if item.endswith('.app'):
+            return os.path.join(applications_dir, item)
+    return None
+
+
+def determine_version(version, product_info):
+    if version:
+        if version == 'latest':
+            from distutils.version import StrictVersion
+            latest_version = StrictVersion('0.0.0')
+            for index, product_id in enumerate(product_info):
+                d = product_info[product_id]['version']
+                if d > latest_version:
+                    latest_version = d
+
+            if latest_version == StrictVersion("0.0.0"):
+                print("Could not find latest version {}")
+                exit(1)
+
+            version = str(latest_version)
+
+        for index, product_id in enumerate(product_info):
+            v = product_info[product_id]['version']
+            if v == version:
+                return product_id, product_info[product_id]['title']
+
+        print("Could not find version {}. Versions available are:".format(version))
+        for _, pid in enumerate(product_info):
+            print("- {}".format(product_info[pid]['version']))
+
+        exit(1)
+
+    # display a menu of choices (some seed catalogs have multiple installers)
+    print('%2s %12s %10s %11s  %s' % ('#', 'ProductID', 'Version',
+                                           'Post Date', 'Title'))
+    for index, product_id in enumerate(product_info):
+        print('%2s %12s %10s %11s  %s' % (
+            index + 1,
+            product_id,
+            product_info[product_id]['version'],
+            product_info[product_id]['PostDate'].strftime('%Y-%m-%d'),
+            product_info[product_id]['title']
+        ))
+
+    answer = input(
+        '\nChoose a product to download (1-%s): ' % len(product_info))
+    try:
+        index = int(answer) - 1
+        if index < 0:
+            raise ValueError
+        product_id = list(product_info.keys())[index]
+        return product_id, product_info[product_id]['title']
+    except (ValueError, IndexError):
+        pass
+
+    print('Invalid input provided.')
+    exit(0)
+
+
+def main():
+    '''Do the main thing here'''
+    """
+    if os.getuid() != 0:
+        sys.exit('This command requires root (to install packages), so please '
+                 'run again with sudo or as root.')
+    """
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--workdir', metavar='path_to_working_dir',
+                        default='.',
+                        help='Path to working directory on a volume with over '
+                             '10G of available space. Defaults to current working '
+                             'directory.')
+    parser.add_argument('--version', metavar='version',
+                        default=None,
+                        help='The version to download in the format of '
+                             '"$major.$minor.$patch", e.g. "10.15.4". Can '
+                             'be "latest" to download the latest version.')
+    parser.add_argument('--compress', action='store_true',
+                        help='Output a read-only compressed disk image with '
+                             'the Install macOS app at the root. This is now the '
+                             'default. Use --raw to get a read-write sparse image '
+                             'with the app in the Applications directory.')
+    parser.add_argument('--raw', action='store_true',
+                        help='Output a read-write sparse image '
+                             'with the app in the Applications directory. Requires '
+                             'less available disk space and is faster.')
+    parser.add_argument('--ignore-cache', action='store_true',
+                        help='Ignore any previously cached files.')
+    args = parser.parse_args()
+
+    su_catalog_url = get_default_catalog()
+    if not su_catalog_url:
+        print('Could not find a default catalog url for this OS version.', file=sys.stderr)
+        exit(-1)
+
+    # download sucatalog and look for products that are for macOS installers
+    catalog = download_and_parse_sucatalog(
+        su_catalog_url, args.workdir, ignore_cache=args.ignore_cache)
+    product_info = os_installer_product_info(
+        catalog, args.workdir, ignore_cache=args.ignore_cache)
+
+    if not product_info:
+        print('No macOS installer products found in the sucatalog.', file=sys.stderr)
+        exit(-1)
+
+    product_id, product_title = determine_version(args.version, product_info)
+    print(product_id, product_title)
+
+    # download all the packages for the selected product
+    replicate_product(catalog, product_id, args.workdir, ignore_cache=args.ignore_cache, product_title=product_title)
+
+
+if __name__ == '__main__':
+    main()

osx-serial-generator

@@ -1 +1 @@
-Subproject commit 0149015547a26b991fc6035972dae75357dc68aa
+Subproject commit 1d7425a7fa929423d965334cd78e9c75aeff2ad0

vnc-version/Dockerfile

@@ -7,7 +7,7 @@
 # 
 # Title:            Mac on Docker (Docker-OSX) [VNC EDITION]
 # Author:           Sick.Codes https://sick.codes/        
-# Version:          3.0
+# Version:          3.1
 # License:          GPLv3+
 # 
 # All credits for OSX-KVM and the rest at Kholia's repo: https://github.com/kholia/osx-kvm
@@ -113,7 +113,7 @@ RUN cat vnc.sh Launch.sh > Launch_custom.sh
 
 RUN chmod +x Launch_custom.sh
 
-RUN tee vncpasswd_file <<< "${VNC_PASSWORD:=$(openssl rand -hex 4)}"
+RUN tee vncpasswd_file <<< "${VNC_PASSWORD:="$(tr -dc '[:graph:]' </dev/urandom | head -c8)"}"
 RUN vncpasswd -f < vncpasswd_file > ${HOME}/.vnc/passwd
 
 RUN chmod 600 ~/.vnc/passwd