diff --git a/.gitignore b/.gitignore index 27b703f..d21c688 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,2 @@ -owl testdata/ -owl.1* +dist/ diff --git a/Makefile b/Makefile index 2b187f7..35e9fc8 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,26 @@ SOURCES=*.go LINKER_FLAGS=-X main.OwlVersion=`git describe --tags --dirty` +INSTALL_DIR=${HOME}/.local + owl: ${SOURCES} go build -o owl -ldflags "${LINKER_FLAGS}" -- ${SOURCES} + owl.1: man.md pandoc -s --shift-heading-level-by=-1 --to=man man.md > owl.1 + build: owl owl.1 + test: *.go go test -v . -.PHONY: build test + +install: + install -m 0755 -D owl ${INSTALL_DIR}/bin/owl + install -m 0644 -D owl.1 ${INSTALL_DIR}/man/man1/owl.1 + install -m 0644 -D LICENSE ${INSTALL_DIR}/share/doc/LICENSE + +uninstall: + rm -I ${INSTALL_DIR}/bin/owl \ + ${INSTALL_DIR}/man/man1/owl.1 \ + ${INSTALL_DIR}/share/doc/LICENSE + +.PHONY: build test install uninstall diff --git a/README.md b/README.md index 484ea3e..d4ad9d6 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,32 @@ appropriate - see `man.md` for usage and other important information. See [Infrequently Asked Questions](./iaq.md) for unimportant information. + +## Installation + +Firstly, bear in mind that any program which renames files could result in +data loss. In particular, while Owl has gone through some basic testing (see +<./test.bash> for details), from my own use on a large nested directory Owl +seems to miss some files, needing repeated use to rename each. This likely +indicates some kind of **severe bug**. + +If you want to use Owl yourself, I would recommend one of the following: +1. Don't use the `--recurse` flag, only use on individual files +2. Use the `--dry-run` flag to find which files need renaming, but rename + files manually. +3. Contribute to Owl and help me fix the bug! +4. Use a different tool. A few alternatives are listed below, though I + haven't used any of them and so can't say whether they are good or not. + +If you still want to install Owl after that, then do the following: +1. Install build dependencies (`make` and the `go` tool) +2. Clone this repository +3. Run `make build` +4. Run `make install` +Note the 3rd step won't work on windows, although you can likely just put +the executable (`./owl`) somewhere on your `PATH` environment variable and +it *should* work just the same. + --- ## Links @@ -37,3 +63,5 @@ See [Infrequently Asked Questions](./iaq.md) for unimportant information. 5. mmv: 6. PathShortener: 7. fuseblk-filename-fixer: +8. POSIX Portable Filename Set (see section 3.265): + diff --git a/main.go b/main.go index 57aa034..d3f6046 100644 --- a/main.go +++ b/main.go @@ -25,6 +25,7 @@ import ( "os" fpath "path/filepath" "slices" + "strconv" "strings" ) @@ -39,6 +40,16 @@ import ( */ type runeset [][2]rune +/** + * A basic search-and-replace operation. Replaces each instance of "Target" + * with each member of "Subs" in order, with all instances beyond + * length(Subs) being replaced by the last member of Subs. + */ +type replacement struct { + Target string + Subs []string +} + // Unrelated to the standard library's "context". type context struct { FileList []string @@ -48,6 +59,8 @@ type context struct { DoHelp bool DoVersion bool Rset runeset + TruncLen int + Replacements []replacement } @@ -76,14 +89,23 @@ var FAT_RUNESET = runeset{ } var POSIX_PORTABLE_RUNESET = runeset{ - { 0x2d, 0x2d }, - { 0x2e, 0x2e }, + { 0x2d, 0x2e }, { 0x5f, 0x5f }, { 0x30, 0x39 }, { 0x41, 0x5a }, { 0x61, 0x7a }, } +var SHELL_RUNESET = runeset{ + { 0x25, 0x25 }, + { 0x2d, 0x2e }, + { 0x30, 0x39 }, + { 0x41, 0x5a }, + { 0x5f, 0x5f }, + { 0x61, 0x7a }, + { 0xc0, MAX_CODE_POINT }, +} + /// MAIN FUNCTIONS /** * Compares paths so that all files come before the directories that contain @@ -189,6 +211,32 @@ func (ctx *context) restrictRuneset(s string) string { return result } +/** + * Truncate to the number of bytes given by the context, but don't break in + * the middle of a rune. + */ +func (ctx *context) truncate(name string) string { + if ctx.TruncLen < 1 { + return name + } + for byteIndex, _ := range name { + if byteIndex > ctx.TruncLen { + return name[:byteIndex] + } + } + return name +} + +func (ctx *context) searchAndReplace(name string) string { + for _, rep := range ctx.Replacements { + for i:=0; i>) for replacement: please use '{TARGET}:{REPLACEMENT1},{REPLACEMENT2},...'", + args[index], + )) + } + result.Replacements = append( + result.Replacements, + replacement{ target, strings.Split(subs, ",") }, + ) + case "-t", "--truncate": + index++ + var err error + result.TruncLen, err = strconv.Atoi(args[index]) + if err != nil || result.TruncLen < 1 { + return result, errors.New(fmt.Sprintf( + "Invalid truncation length: <<%s>>", + args[index], + )) + } case "-h", "--help": result.DoHelp = true case "-v", "--version": @@ -256,13 +343,30 @@ func printHelp() { Rename FILES such that all characters that are invalid in FAT file systems (?,\,*,etc.) are removed. - - Options: - -s,--strategy - -h,--help - -v,--version - -r,--recurse DIRECTORY + Options: + -r,--recurse DIRECTORY + Recursively search for files in DIRECTORY with bad characters + -s,--strategy remove|represent + What to do with bad characters; either replace with a representation or + remove + -e,--valid-set fat|posix|shell + Select a set of characters to limit filenames to. + -p,--portable + Alias for "--valid-set posix" + -t,--truncate LENGTH + Truncate all filenames to at most LENGTH bytes + -c,--replace TARGET:REPLACEMENT1,REPLACEMENT2,... + Replace each instance of TARGET in file names with REPLACEMENT1 the + first time they appear, REPLACEMENT2 the second time, and so on + -n,--dry-run + Do not rename anything, just print what would be done + -h,--help + Show this message + -v,--version + Show version + + See man page for more details `); } @@ -290,9 +394,9 @@ func main() { // Calculate the new name oldName := fpath.Base(file) dirName := fpath.Dir(file) - newName := ctx.restrictRuneset( + newName := ctx.truncate(ctx.restrictRuneset(ctx.searchAndReplace( strings.ToValidUTF8(oldName, "_INVALID_"), - ) + ))) newPath := fpath.Join(dirName, newName) // Check that we want to rename this file if newPath == file { diff --git a/man.md b/man.md index 8d2ab6b..df7e52f 100644 --- a/man.md +++ b/man.md @@ -1,4 +1,4 @@ -# owl(1) +# OWL(1) *The moon is bright. The owls are ready for the hunt.* @@ -6,7 +6,7 @@ Owl is a file renaming tool; it removes character from file names which are not compatible with `FAT` file systems (typically `FAT32` or `exFAT`). ``` -owl COMMAND OPTIONS FILE1 FILE2 ... +owl OPTIONS FILE1 FILE2 ... ``` Rename `FILE`s given at the command line such that all `FAT`-incompatible @@ -15,7 +15,11 @@ characters are removed. By default it replaces invalid characters with flag below. -## Options: +## OPTIONS: +### -r, \-\-recurse DIRECTORY +Recursively search DIRECTORY for files/directories to rename. Search +includes DIRECTORY itself. + ### -s, \-\-strategy STRATEGY Change what happens to invalid characters. Choices are: @@ -24,13 +28,38 @@ Change what happens to invalid characters. Choices are: - "represent": replace each character with "\_Unum_", where "num" is the Unicode Code Point of the character. -### -r, \-\-recurse DIRECTORY -Recursively search DIRECTORY for files/directories to rename. Search -includes DIRECTORY itself. +### -e,\-\-valid-set fat|posix|shell +Select which characters are considered "invalid" or "bad". Options are: + - fat: Characters which are valid in file names on FAT file systems. This + is the default. + - posix: The POSIX Portable Filename Character Set `A-Za-z0-9.-_` + - shell: Characters that *should* not need to be quoted when used in a + shell, i.e. all characters apart from control characters, whitespace, and + most punctuation. + +### -p,\-\-portable +An alias for "\-\-valid-set posix". + +### -t,\-\-truncate LENGTH +Truncate all given file names to at most `LENGTH` bytes. Note that if you +have file names with larger Unicode code points (like CJK characters), you +may get unexpected results with this option, though it will not cut the file +name part way through a character. + +### -c,\-\-replace TARGET:REPLACEMENT1,REPLACEMENT2,... +Remove every instance of the string `TARGET` in every file name and replace +the first instance with `REPLACEMENT1`, the second with `REPLACEMENT2`, and +so on. If there are N `REPLACEMENT`'s and more than N instances of `TARGET` +in a file name, all instances after the Nth are replaced by the last +replacement. Replacements can be empty. See EXAMPLES below for this one. + +Remember to quote `TARGET:REPLACEMENT1,...` so that your shell doesn't +misinterpret the colon. ### -n, \-\-dry-run Causes Owl to just print a representation of what would be done without -actually renaming any files. +actually renaming any files. Checks for name collisions with existing files +or other renaming operations are still done. ### -v, \-\-version Show version information. @@ -39,7 +68,66 @@ Show version information. Show a small help message. -## Edge Cases & Other Tidbits +## EXAMPLES +Suppose we have a file called +``` +PodEp 37: Why are there so many spaces in this file name? +``` +in the current working directory. We will refer to it as FILE Then: + +- The command + ``` + owl --replace ' :,-,_' FILE + ``` + will rename FILE to + `PodEp37:-Why_are_there_so_many_spaces_in_this_file_name_U3F_` +- The command + ``` + owl --truncate 48 --replace ' :,-,_' FILE + ``` + will rename FILE to + `PodEp37:-Why_are_there_so_many_spaces_in_this_fi` +- The command + ``` + owl --portable --strategy remove --truncate 48 FILE + ``` + will rename FILE to + `PodEp37:Whyaretheresomanyspacesinthisfilename` + +Now suppose we have files "IMPORTANT FILE?" and "Important File". +- If you run + ``` + owl --strategy remove "IMPORTANT FILE?" + ``` + owl will just return an error, saying that the new name would collide with + "important file". +- If you run + ``` + owl --strategy remove --valid-set posix "IMPORTANT FILE?" "Important File" + ``` + then owl will will rename one file, but refuse to rename the other, as it + would conflict with the now-renamed first file. + Note you can use the `--dry-run` flag to check for collisions like this + before renaming anything. + + +## EDGE CASES & OTHER TIDBITS +### What order does owl truncate, remove characters, etc.? +Owl does things in the following order: + +1. Remove invalid UTF-8 +2. Replace stuff (see `--replace` flag) +3. Deal with invalid characters (see `--strategy` flag) +4. Truncate file name + +The order in which you use the flags does not change this. + +### What if the new name Owl chooses for a file already exists? +Owl checks before each renaming operation to see if the new name already +exists, and skips that file if it does. The check is also case-insensitive. +See [EXAMPLES](#examples) for more details. + + ### What doesn't FAT allow? Mainly `*<>\|/:?'` @@ -68,6 +156,26 @@ Owl has only been tested so far on Linux with a UTF-8 locale. UTF-16 support on Windows is likely possible, but has not been tested. Please contact me if you would like Windows support (see below). +### What about other restrictions on file names? +Windows (and by extension FAT file systems, which come from Microsoft) may +have other restrictions on file names, like specific disallowed names like +"CON" or names which ends with a ".". Owl does not check for such things; +you should look for those using a tool like `find`. + +For completeness, the disallowed names are +``` +CON,AUX,COM1,COM2COM3,COM4,LPT1,LPT2,LPT3,PRN,NUL +``` + +### What about symbolic links (a.k.a. symlinks)? +Owl will rename the *link*, but not the file that the link points to, even +if the link is broken. If you are having issues with broken symbolic links +and are on a POSIX/UNIX-like system, you can use +``` +find -L DIRECTORY -type l +``` +to find all broken links in DIRECTORY. + ## AUTHORS & COPYRIGHT Written by user SixteenThousand of github.com (email diff --git a/test.bash b/test.bash index 6f41fc7..acf8cd3 100755 --- a/test.bash +++ b/test.bash @@ -42,6 +42,10 @@ owltest_setup() { $'No\u0338rse\u0301' \ 'Question? Why' \ 'QUESTION: WHY' + ln -s "${PWD}/::>?" ascii_dir/file + ln -s "${PWD}/::>?" ascii_dir/fi?le + ln -s "${PWD}/no-change-needed" 'ascii_dir/oh?' + ln -s /invalid/path 'ascii_dir/bad link' } owltest_teardown() {