From 00f8ca515f365ce2a280fd5c3c2e6f459e0a4d2c Mon Sep 17 00:00:00 2001 From: Thomas Lindop Date: Tue, 27 May 2025 23:21:48 +0100 Subject: [PATCH 01/10] >> update documentation --- README.md | 2 ++ main.go | 2 ++ man.md | 5 +++++ 3 files changed, 9 insertions(+) diff --git a/README.md b/README.md index 484ea3e..e9fb106 100644 --- a/README.md +++ b/README.md @@ -37,3 +37,5 @@ See [Infrequently Asked Questions](./iaq.md) for unimportant information. 5. mmv: 6. PathShortener: 7. fuseblk-filename-fixer: +8. POSIX Portable Filename Set (see section 3.265): + diff --git a/main.go b/main.go index 57aa034..12b70f3 100644 --- a/main.go +++ b/main.go @@ -262,6 +262,8 @@ func printHelp() { -h,--help -v,--version -r,--recurse DIRECTORY + -n,--dry-run + -p,--portable `); } diff --git a/man.md b/man.md index 8d2ab6b..02b11c2 100644 --- a/man.md +++ b/man.md @@ -32,6 +32,11 @@ includes DIRECTORY itself. Causes Owl to just print a representation of what would be done without actually renaming any files. +### -p,\-\-portable +Restrict characters to ones in the POSIX Portable Filename Character Set +(see , section 3.265) +instead of FAT compatible ones. + ### -v, \-\-version Show version information. From f871153f2973a4f33bbfb41593ad4671d122cef7 Mon Sep 17 00:00:00 2001 From: Thomas Lindop Date: Tue, 27 May 2025 23:45:09 +0100 Subject: [PATCH 02/10] >> add truncate flag --- main.go | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index 12b70f3..32ce07c 100644 --- a/main.go +++ b/main.go @@ -25,6 +25,7 @@ import ( "os" fpath "path/filepath" "slices" + "strconv" "strings" ) @@ -48,6 +49,7 @@ type context struct { DoHelp bool DoVersion bool Rset runeset + TruncLen int } @@ -189,6 +191,22 @@ func (ctx *context) restrictRuneset(s string) string { return result } +/** + * Truncate to the number of bytes given by the context, but don't break in + * the middle of a rune. + */ +func (ctx *context) truncate(name string) string { + if ctx.TruncLen < 1 { + return name + } + for byteIndex, _ := range name { + if byteIndex > ctx.TruncLen { + return name[:byteIndex] + } + } + return name +} + func warn(msg string) { fmt.Fprintf(os.Stderr, "\x1b[33m%s\x1b[0m\n", msg) } @@ -211,6 +229,7 @@ func parseCLIArgs(args []string) (context, error) { DoHelp: false, DoVersion: false, Rset: FAT_RUNESET, + TruncLen: 0, } index := 1 isFlag := func(arg string) bool { @@ -228,6 +247,16 @@ func parseCLIArgs(args []string) (context, error) { result.Strategy = args[index] case "-p", "--portable": result.Rset = POSIX_PORTABLE_RUNESET + case "-t", "--truncate": + index++ + var err error + result.TruncLen, err = strconv.Atoi(args[index]) + if err != nil || result.TruncLen < 1 { + return result, errors.New(fmt.Sprintf( + "Invalid truncation length: <<%s>>", + args[index], + )) + } case "-h", "--help": result.DoHelp = true case "-v", "--version": @@ -292,9 +321,9 @@ func main() { // Calculate the new name oldName := fpath.Base(file) dirName := fpath.Dir(file) - newName := ctx.restrictRuneset( + newName := ctx.truncate(ctx.restrictRuneset( strings.ToValidUTF8(oldName, "_INVALID_"), - ) + )) newPath := fpath.Join(dirName, newName) // Check that we want to rename this file if newPath == file { From 5232b8fb3b7e08136c38c8fb2b1c4970403d7e58 Mon Sep 17 00:00:00 2001 From: Thomas Lindop Date: Wed, 28 May 2025 00:32:08 +0100 Subject: [PATCH 03/10] >> add basic search and replace --- main.go | 39 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index 32ce07c..5a021d9 100644 --- a/main.go +++ b/main.go @@ -40,6 +40,16 @@ import ( */ type runeset [][2]rune +/** + * A basic search-and-replace operation. Replaces each instance of "Target" + * with each member of "Subs" in order, with all instances beyond + * length(Subs) being replaced by the last member of Subs. + */ +type replacement struct { + Target string + Subs []string +} + // Unrelated to the standard library's "context". type context struct { FileList []string @@ -50,6 +60,7 @@ type context struct { DoVersion bool Rset runeset TruncLen int + Replacements []replacement } @@ -207,6 +218,16 @@ func (ctx *context) truncate(name string) string { return name } +func (ctx *context) searchAndReplace(name string) string { + for _, rep := range ctx.Replacements { + for i:=0; i>) for replacement: please use '{TARGET}:{REPLACEMNT1},{REPLACEMNT2},...'", + args[index], + )) + } + result.Replacements = append( + result.Replacements, + replacement{ target, strings.Split(subs, ",") }, + ) case "-t", "--truncate": index++ var err error @@ -321,9 +356,9 @@ func main() { // Calculate the new name oldName := fpath.Base(file) dirName := fpath.Dir(file) - newName := ctx.truncate(ctx.restrictRuneset( + newName := ctx.truncate(ctx.restrictRuneset(ctx.searchAndReplace( strings.ToValidUTF8(oldName, "_INVALID_"), - )) + ))) newPath := fpath.Join(dirName, newName) // Check that we want to rename this file if newPath == file { From 03d4c62e61ea0e7ece11b14302fbbc0588df5ef0 Mon Sep 17 00:00:00 2001 From: Thomas Lindop Date: Wed, 28 May 2025 10:58:55 +0100 Subject: [PATCH 04/10] >> add shell compatible runeset > add flag to select runeset > update documentation --- main.go | 43 +++++++++++++++++++++++++++++++++++++++---- man.md | 13 ++++++++++--- 2 files changed, 49 insertions(+), 7 deletions(-) diff --git a/main.go b/main.go index 5a021d9..c5b78a2 100644 --- a/main.go +++ b/main.go @@ -89,14 +89,23 @@ var FAT_RUNESET = runeset{ } var POSIX_PORTABLE_RUNESET = runeset{ - { 0x2d, 0x2d }, - { 0x2e, 0x2e }, + { 0x2d, 0x2e }, { 0x5f, 0x5f }, { 0x30, 0x39 }, { 0x41, 0x5a }, { 0x61, 0x7a }, } +var SHELL_RUNESET = runeset{ + { 0x25, 0x25 }, + { 0x2d, 0x2e }, + { 0x30, 0x39 }, + { 0x41, 0x5a }, + { 0x5f, 0x5f }, + { 0x61, 0x7a }, + { 0xc0, MAX_CODE_POINT }, +} + /// MAIN FUNCTIONS /** * Compares paths so that all files come before the directories that contain @@ -269,12 +278,26 @@ func parseCLIArgs(args []string) (context, error) { result.Strategy = args[index] case "-p", "--portable": result.Rset = POSIX_PORTABLE_RUNESET + case "-e", "--valid-set": + index++ + switch args[index] { + case "fat": + result.Rset = FAT_RUNESET + case "posix": + result.Rset = POSIX_PORTABLE_RUNESET + case "shell": + result.Rset = SHELL_RUNESET + default: + return result, errors.New( + "Invalid character set! Please choose one of fat,posix,shell", + ) + } case "-c", "--replace": index++ target, subs, ok := strings.Cut(args[index], ":") if !ok { return result, errors.New(fmt.Sprintf( - "Incorrect syntax (<<%s>>) for replacement: please use '{TARGET}:{REPLACEMNT1},{REPLACEMNT2},...'", + "Incorrect syntax (<<%s>>) for replacement: please use '{TARGET}:{REPLACEMENT1},{REPLACEMENT2},...'", args[index], )) } @@ -322,13 +345,25 @@ func printHelp() { (?,\,*,etc.) are removed. Options: - -s,--strategy + -s,--strategy remove|represent + What to do with bad characters; either replace with a representation or + remove -h,--help + Show this message -v,--version + Show version -r,--recurse DIRECTORY + Recursively search for files in DIRECTORY with bad characters -n,--dry-run + Do not rename anything, just print what would be done + -t,--truncate LENGTH + Truncate all filenames to at most LENGTH bytes + -e,--valid-set fat|posix|shell + Select a set of characters to limit filenames to. -p,--portable + Alias for "--valid-set posix" + See man page for more details `); } diff --git a/man.md b/man.md index 02b11c2..22d87b7 100644 --- a/man.md +++ b/man.md @@ -32,10 +32,17 @@ includes DIRECTORY itself. Causes Owl to just print a representation of what would be done without actually renaming any files. +### -e,\-\-valid-set fat|posix|shell +Select which characters are considered "invalid" or "bad". Options are: + - fat: Characters which are valid in file names on FAT file systems. This + is the default. + - posix: The POSIX Portable Filename Character Set `A-Za-z0-9.-_` + - shell: Characters that *should* not need to be quoted when used in a + shell, i.e. all characters apart from control characters, whitespace, and + most punctuation. + ### -p,\-\-portable -Restrict characters to ones in the POSIX Portable Filename Character Set -(see , section 3.265) -instead of FAT compatible ones. +An alias for "\-\-valid-set posix". ### -v, \-\-version Show version information. From ed919acff7e5e7e7c58785629386ca7db9af9f53 Mon Sep 17 00:00:00 2001 From: Thomas Lindop Date: Wed, 28 May 2025 11:16:30 +0100 Subject: [PATCH 05/10] >> add examples of symbolic links to tests --- test.bash | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test.bash b/test.bash index 6f41fc7..acf8cd3 100755 --- a/test.bash +++ b/test.bash @@ -42,6 +42,10 @@ owltest_setup() { $'No\u0338rse\u0301' \ 'Question? Why' \ 'QUESTION: WHY' + ln -s "${PWD}/::>?" ascii_dir/file + ln -s "${PWD}/::>?" ascii_dir/fi?le + ln -s "${PWD}/no-change-needed" 'ascii_dir/oh?' + ln -s /invalid/path 'ascii_dir/bad link' } owltest_teardown() { From b99b5bc3b7f9e9cf601ee349af5aa3afd41fa8e8 Mon Sep 17 00:00:00 2001 From: Thomas Lindop Date: Wed, 28 May 2025 11:23:58 +0100 Subject: [PATCH 06/10] >> add replace flag to help message > fix whitespace issues in help message --- main.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/main.go b/main.go index c5b78a2..4caa3bc 100644 --- a/main.go +++ b/main.go @@ -343,11 +343,11 @@ func printHelp() { Rename FILES such that all characters that are invalid in FAT file systems (?,\,*,etc.) are removed. - + Options: -s,--strategy remove|represent What to do with bad characters; either replace with a representation or - remove + remove -h,--help Show this message -v,--version @@ -362,8 +362,11 @@ func printHelp() { Select a set of characters to limit filenames to. -p,--portable Alias for "--valid-set posix" + -c,--replace TARGET:REPLACEMENT1,REPLACEMENT2,... + Replace each instance of TARGET in file names with REPLACEMENT1 the + first time they appear, REPLACEMENT2 the second time, and so on - See man page for more details + See man page for more details `); } From 18f16bae58978e97ff5ab6e752fc516b1c72775e Mon Sep 17 00:00:00 2001 From: Thomas Lindop Date: Wed, 28 May 2025 12:16:43 +0100 Subject: [PATCH 07/10] >> complete documentation > reorder list of flags to be consistent --- main.go | 20 +++++----- man.md | 118 ++++++++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 117 insertions(+), 21 deletions(-) diff --git a/main.go b/main.go index 4caa3bc..d3f6046 100644 --- a/main.go +++ b/main.go @@ -345,26 +345,26 @@ func printHelp() { (?,\,*,etc.) are removed. Options: + -r,--recurse DIRECTORY + Recursively search for files in DIRECTORY with bad characters -s,--strategy remove|represent What to do with bad characters; either replace with a representation or remove - -h,--help - Show this message - -v,--version - Show version - -r,--recurse DIRECTORY - Recursively search for files in DIRECTORY with bad characters - -n,--dry-run - Do not rename anything, just print what would be done - -t,--truncate LENGTH - Truncate all filenames to at most LENGTH bytes -e,--valid-set fat|posix|shell Select a set of characters to limit filenames to. -p,--portable Alias for "--valid-set posix" + -t,--truncate LENGTH + Truncate all filenames to at most LENGTH bytes -c,--replace TARGET:REPLACEMENT1,REPLACEMENT2,... Replace each instance of TARGET in file names with REPLACEMENT1 the first time they appear, REPLACEMENT2 the second time, and so on + -n,--dry-run + Do not rename anything, just print what would be done + -h,--help + Show this message + -v,--version + Show version See man page for more details `); diff --git a/man.md b/man.md index 22d87b7..ae8820e 100644 --- a/man.md +++ b/man.md @@ -6,7 +6,7 @@ Owl is a file renaming tool; it removes character from file names which are not compatible with `FAT` file systems (typically `FAT32` or `exFAT`). ``` -owl COMMAND OPTIONS FILE1 FILE2 ... +owl OPTIONS FILE1 FILE2 ... ``` Rename `FILE`s given at the command line such that all `FAT`-incompatible @@ -15,7 +15,11 @@ characters are removed. By default it replaces invalid characters with flag below. -## Options: +## OPTIONS: +### -r, \-\-recurse DIRECTORY +Recursively search DIRECTORY for files/directories to rename. Search +includes DIRECTORY itself. + ### -s, \-\-strategy STRATEGY Change what happens to invalid characters. Choices are: @@ -24,14 +28,6 @@ Change what happens to invalid characters. Choices are: - "represent": replace each character with "\_Unum_", where "num" is the Unicode Code Point of the character. -### -r, \-\-recurse DIRECTORY -Recursively search DIRECTORY for files/directories to rename. Search -includes DIRECTORY itself. - -### -n, \-\-dry-run -Causes Owl to just print a representation of what would be done without -actually renaming any files. - ### -e,\-\-valid-set fat|posix|shell Select which characters are considered "invalid" or "bad". Options are: - fat: Characters which are valid in file names on FAT file systems. This @@ -44,6 +40,27 @@ Select which characters are considered "invalid" or "bad". Options are: ### -p,\-\-portable An alias for "\-\-valid-set posix". +### -t,\-\-truncate LENGTH +Truncate all given file names to at most `LENGTH` bytes. Note that if you +have file names with larger Unicode code points (like CJK characters), you +may get unexpected results with this option, though it will not cut the file +name part way through a character. + +### -c,\-\-replace TARGET:REPLACEMENT1,REPLACEMENT2,... +Remove every instance of the string `TARGET` in every file name and replace +the first instance with `REPLACEMENT1`, the second with `REPLACEMENT2`, and +so on. If there are N `REPLACEMENT`'s and more than N instances of `TARGET` +in a file name, all instances after the Nth are replaced by the last +replacement. Replacements can be empty. See EXAMPLES below for this one. + +Remember to quote `TARGET:REPLACEMENT1,...` so that your shell doesn't +misinterpret the colon. + +### -n, \-\-dry-run +Causes Owl to just print a representation of what would be done without +actually renaming any files. Checks for name collisions with existing files +or other renaming operations are still done. + ### -v, \-\-version Show version information. @@ -51,7 +68,66 @@ Show version information. Show a small help message. -## Edge Cases & Other Tidbits +## EXAMPLES +Suppose we have a file called +``` +PodEp 37: Why are there so many spaces in this file name? +``` +in the current working directory. We will refer to it as FILE Then: + +- The command + ``` + owl --replace ' :,-,_' FILE + ``` + will rename FILE to + `PodEp37:-Why_are_there_so_many_spaces_in_this_file_name_U3F_` +- The command + ``` + owl --truncate 48 --replace ' :,-,_' FILE + ``` + will rename FILE to + `PodEp37:-Why_are_there_so_many_spaces_in_this_fi` +- The command + ``` + owl --portable --strategy remove --truncate 48 FILE + ``` + will rename FILE to + `PodEp37:Whyaretheresomanyspacesinthisfilename` + +Now suppose we have files "IMPORTANT FILE?" and "Important File". +- If you run + ``` + owl --strategy remove "IMPORTANT FILE?" + ``` + owl will just return an error, saying that the new name would collide with + "important file". +- If you run + ``` + owl --strategy remove --valid-set posix "IMPORTANT FILE?" "Important File" + ``` + then owl will will rename one file, but refuse to rename the other, as it + would conflict with the now-renamed first file. + Note you can use the `--dry-run` flag to check for collisions like this + before renaming anything. + + +## EDGE CASES & OTHER TIDBITS +### What order does owl truncate, remove characters, etc.? +Owl does things in the following order: + +1. Remove invalid UTF-8 +2. Replace stuff (see `--replace` flag) +3. Deal with invalid characters (see `--strategy` flag) +4. Truncate file name + +The order in which you use the flags does not change this. + +### What if the new name Owl chooses for a file already exists? +Owl checks before each renaming operation to see if the new name already +exists, and skips that file if it does. The check is also case-insensitive. +See [EXAMPLES](#examples) for more details. + + ### What doesn't FAT allow? Mainly `*<>\|/:?'` @@ -80,6 +156,26 @@ Owl has only been tested so far on Linux with a UTF-8 locale. UTF-16 support on Windows is likely possible, but has not been tested. Please contact me if you would like Windows support (see below). +### What about other restrictions on file names? +Windows (and by extension FAT file systems, which come from Microsoft) may +have other restrictions on file names, like specific disallowed names like +"CON" or names which ends with a ".". Owl does not check for such things; +you should look for those using a tool like `find`. + +For completeness, the disallowed names are +``` +CON,AUX,COM1,COM2COM3,COM4,LPT1,LPT2,LPT3,PRN,NUL +``` + +### What about symbolic links (a.k.a. symlinks)? +Owl will rename the *link*, but not the file that the link points to, even +if the link is broken. If you are having issues with broken symbolic links +and are on a POSIX/UNIX-like system, you can use +``` +find -L DIRECTORY -type l +``` +to find all broken links in DIRECTORY. + ## AUTHORS & COPYRIGHT Written by user SixteenThousand of github.com (email From b20e1286d6d18ae85133dec603667223e2ce821b Mon Sep 17 00:00:00 2001 From: Thomas Lindop Date: Wed, 28 May 2025 12:27:19 +0100 Subject: [PATCH 08/10] >> make install recipes > fix title in man page --- Makefile | 15 ++++++++++++++- man.md | 2 +- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 2b187f7..64c685e 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,11 @@ SOURCES=*.go LINKER_FLAGS=-X main.OwlVersion=`git describe --tags --dirty` +ifeq (${XDG_DATA_HOME},) + XDG_DATA_HOME=${HOME}/.local/share +endif +MAN_DIR=${XDG_DATA_HOME}/man/man1 +BIN_DIR=${HOME}/.local/bin + owl: ${SOURCES} go build -o owl -ldflags "${LINKER_FLAGS}" -- ${SOURCES} owl.1: man.md @@ -7,4 +13,11 @@ owl.1: man.md build: owl owl.1 test: *.go go test -v . -.PHONY: build test +install: build + install -m 0777 owl ${BIN_DIR} + mkdir -p ${MAN_DIR} + install owl.1 ${MAN_DIR} +uninstall: + rm -I ${BIN_DIR}/owl ${MAN_DIR}/owl.1 + +.PHONY: build test install uninstall diff --git a/man.md b/man.md index ae8820e..df7e52f 100644 --- a/man.md +++ b/man.md @@ -1,4 +1,4 @@ -# owl(1) +# OWL(1) *The moon is bright. The owls are ready for the hunt.* From c45d3bda9ad703c4d3b8997426c28b41fb8e35db Mon Sep 17 00:00:00 2001 From: Thomas Lindop Date: Mon, 2 Jun 2025 10:14:55 +0100 Subject: [PATCH 09/10] >> add installation section to README > make note of bug --- README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/README.md b/README.md index e9fb106..d4ad9d6 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,32 @@ appropriate - see `man.md` for usage and other important information. See [Infrequently Asked Questions](./iaq.md) for unimportant information. + +## Installation + +Firstly, bear in mind that any program which renames files could result in +data loss. In particular, while Owl has gone through some basic testing (see +<./test.bash> for details), from my own use on a large nested directory Owl +seems to miss some files, needing repeated use to rename each. This likely +indicates some kind of **severe bug**. + +If you want to use Owl yourself, I would recommend one of the following: +1. Don't use the `--recurse` flag, only use on individual files +2. Use the `--dry-run` flag to find which files need renaming, but rename + files manually. +3. Contribute to Owl and help me fix the bug! +4. Use a different tool. A few alternatives are listed below, though I + haven't used any of them and so can't say whether they are good or not. + +If you still want to install Owl after that, then do the following: +1. Install build dependencies (`make` and the `go` tool) +2. Clone this repository +3. Run `make build` +4. Run `make install` +Note the 3rd step won't work on windows, although you can likely just put +the executable (`./owl`) somewhere on your `PATH` environment variable and +it *should* work just the same. + --- ## Links From 9c34a31fdef3046b5b9e3d8791e400ec98788337 Mon Sep 17 00:00:00 2001 From: Thomas Lindop Date: Tue, 22 Jul 2025 11:07:09 +0100 Subject: [PATCH 10/10] >> add install recipes --- .gitignore | 3 +-- Makefile | 23 +++++++++++++---------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/.gitignore b/.gitignore index 27b703f..d21c688 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,2 @@ -owl testdata/ -owl.1* +dist/ diff --git a/Makefile b/Makefile index 64c685e..35e9fc8 100644 --- a/Makefile +++ b/Makefile @@ -1,23 +1,26 @@ SOURCES=*.go LINKER_FLAGS=-X main.OwlVersion=`git describe --tags --dirty` -ifeq (${XDG_DATA_HOME},) - XDG_DATA_HOME=${HOME}/.local/share -endif -MAN_DIR=${XDG_DATA_HOME}/man/man1 -BIN_DIR=${HOME}/.local/bin +INSTALL_DIR=${HOME}/.local owl: ${SOURCES} go build -o owl -ldflags "${LINKER_FLAGS}" -- ${SOURCES} + owl.1: man.md pandoc -s --shift-heading-level-by=-1 --to=man man.md > owl.1 + build: owl owl.1 + test: *.go go test -v . -install: build - install -m 0777 owl ${BIN_DIR} - mkdir -p ${MAN_DIR} - install owl.1 ${MAN_DIR} + +install: + install -m 0755 -D owl ${INSTALL_DIR}/bin/owl + install -m 0644 -D owl.1 ${INSTALL_DIR}/man/man1/owl.1 + install -m 0644 -D LICENSE ${INSTALL_DIR}/share/doc/LICENSE + uninstall: - rm -I ${BIN_DIR}/owl ${MAN_DIR}/owl.1 + rm -I ${INSTALL_DIR}/bin/owl \ + ${INSTALL_DIR}/man/man1/owl.1 \ + ${INSTALL_DIR}/share/doc/LICENSE .PHONY: build test install uninstall