Merge pull request #2392 from aledbf/fix-fsnotify

Ensure dep fix fsnotify
This commit is contained in:
k8s-ci-robot 2018-04-21 00:16:00 -07:00 committed by GitHub
commit b8ab2b4fd4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
26 changed files with 290 additions and 79 deletions

8
Gopkg.lock generated
View file

@ -492,10 +492,10 @@
version = "v1.7.2" version = "v1.7.2"
[[projects]] [[projects]]
name = "gopkg.in/fsnotify.v1" name = "gopkg.in/fsnotify/fsnotify.v1"
packages = ["."] packages = ["."]
revision = "629574ca2a5df945712d3079857300b5e4da0236" revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9"
version = "v1.4.2" version = "v1.4.7"
[[projects]] [[projects]]
name = "gopkg.in/go-playground/pool.v3" name = "gopkg.in/go-playground/pool.v3"
@ -865,6 +865,6 @@
[solve-meta] [solve-meta]
analyzer-name = "dep" analyzer-name = "dep"
analyzer-version = 1 analyzer-version = 1
inputs-digest = "262bd1cf8d4735c8d09a6f6d83fed25ad9e1478443a7f6368210e7c3fbb58977" inputs-digest = "b73a300a6dbcaac85d84661c5bb1a29e86fbad84ba2c8676ab71f554bf61c2b0"
solver-name = "gps-cdcl" solver-name = "gps-cdcl"
solver-version = 1 solver-version = 1

View file

@ -20,6 +20,11 @@
# name = "github.com/x/y" # name = "github.com/x/y"
# version = "2.4.0" # version = "2.4.0"
[prune]
non-go = true
go-tests = true
unused-packages = true
[[override]] [[override]]
name = "github.com/docker/distribution" name = "github.com/docker/distribution"
revision = "edc3ab29cdff8694dd6feb85cfeb4b5f1b38ed9c" revision = "edc3ab29cdff8694dd6feb85cfeb4b5f1b38ed9c"
@ -78,7 +83,7 @@
version = "1.0.1" version = "1.0.1"
[[constraint]] [[constraint]]
name = "gopkg.in/fsnotify.v1" name = "gopkg.in/fsnotify/fsnotify.v1"
version = "1.4.2" version = "1.4.2"
[[constraint]] [[constraint]]

View file

@ -197,3 +197,9 @@ docker-push: all-push
.PHONY: check_dead_links .PHONY: check_dead_links
check_dead_links: check_dead_links:
docker run -t -v $$PWD:/tmp aledbf/awesome_bot:0.1 --allow-dupe --allow-redirect $(shell find $$PWD -mindepth 1 -name "*.md" -printf '%P\n' | grep -v vendor | grep -v Changelog.md) docker run -t -v $$PWD:/tmp aledbf/awesome_bot:0.1 --allow-dupe --allow-redirect $(shell find $$PWD -mindepth 1 -name "*.md" -printf '%P\n' | grep -v vendor | grep -v Changelog.md)
.PHONY: dep-ensure
dep-ensure:
dep version || go get -u github.com/golang/dep/cmd/dep
dep ensure -v
dep prune -v

View file

@ -21,7 +21,7 @@ import (
"path" "path"
"strings" "strings"
"gopkg.in/fsnotify.v1" "gopkg.in/fsnotify/fsnotify.v1"
) )
// FileWatcher is an interface we use to watch changes in files // FileWatcher is an interface we use to watch changes in files

View file

@ -2,12 +2,14 @@ sudo: false
language: go language: go
go: go:
- 1.6.3 - 1.8.x
- 1.9.x
- tip - tip
matrix: matrix:
allow_failures: allow_failures:
- go: tip - go: tip
fast_finish: true
before_script: before_script:
- go get -u github.com/golang/lint/golint - go get -u github.com/golang/lint/golint

View file

@ -8,8 +8,10 @@
# Please keep the list sorted. # Please keep the list sorted.
Aaron L <aaron@bettercoder.net>
Adrien Bustany <adrien@bustany.org> Adrien Bustany <adrien@bustany.org>
Amit Krishnan <amit.krishnan@oracle.com> Amit Krishnan <amit.krishnan@oracle.com>
Anmol Sethi <me@anmol.io>
Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com>
Bruno Bigras <bigras.bruno@gmail.com> Bruno Bigras <bigras.bruno@gmail.com>
Caleb Spare <cespare@gmail.com> Caleb Spare <cespare@gmail.com>
@ -26,6 +28,7 @@ Kelvin Fo <vmirage@gmail.com>
Ken-ichirou MATSUZAWA <chamas@h4.dion.ne.jp> Ken-ichirou MATSUZAWA <chamas@h4.dion.ne.jp>
Matt Layher <mdlayher@gmail.com> Matt Layher <mdlayher@gmail.com>
Nathan Youngman <git@nathany.com> Nathan Youngman <git@nathany.com>
Nickolai Zeldovich <nickolai@csail.mit.edu>
Patrick <patrick@dropbox.com> Patrick <patrick@dropbox.com>
Paul Hammond <paul@paulhammond.org> Paul Hammond <paul@paulhammond.org>
Pawel Knap <pawelknap88@gmail.com> Pawel Knap <pawelknap88@gmail.com>
@ -33,12 +36,15 @@ Pieter Droogendijk <pieter@binky.org.uk>
Pursuit92 <JoshChase@techpursuit.net> Pursuit92 <JoshChase@techpursuit.net>
Riku Voipio <riku.voipio@linaro.org> Riku Voipio <riku.voipio@linaro.org>
Rob Figueiredo <robfig@gmail.com> Rob Figueiredo <robfig@gmail.com>
Rodrigo Chiossi <rodrigochiossi@gmail.com>
Slawek Ligus <root@ooz.ie> Slawek Ligus <root@ooz.ie>
Soge Zhang <zhssoge@gmail.com> Soge Zhang <zhssoge@gmail.com>
Tiffany Jernigan <tiffany.jernigan@intel.com> Tiffany Jernigan <tiffany.jernigan@intel.com>
Tilak Sharma <tilaks@google.com> Tilak Sharma <tilaks@google.com>
Tom Payne <twpayne@gmail.com>
Travis Cline <travis.cline@gmail.com> Travis Cline <travis.cline@gmail.com>
Tudor Golubenco <tudor.g@gmail.com> Tudor Golubenco <tudor.g@gmail.com>
Vahe Khachikyan <vahe@live.ca>
Yukang <moorekang@gmail.com> Yukang <moorekang@gmail.com>
bronze1man <bronze1man@gmail.com> bronze1man <bronze1man@gmail.com>
debrando <denis.brandolini@gmail.com> debrando <denis.brandolini@gmail.com>

View file

@ -1,5 +1,15 @@
# Changelog # Changelog
## v1.4.7 / 2018-01-09
* BSD/macOS: Fix possible deadlock on closing the watcher on kqueue (thanks @nhooyr and @glycerine)
* Tests: Fix missing verb on format string (thanks @rchiossi)
* Linux: Fix deadlock in Remove (thanks @aarondl)
* Linux: Watch.Add improvements (avoid race, fix consistency, reduce garbage) (thanks @twpayne)
* Docs: Moved FAQ into the README (thanks @vahe)
* Linux: Properly handle inotify's IN_Q_OVERFLOW event (thanks @zeldovich)
* Docs: replace references to OS X with macOS
## v1.4.2 / 2016-10-10 ## v1.4.2 / 2016-10-10
* Linux: use InotifyInit1 with IN_CLOEXEC to stop leaking a file descriptor to a child process when using fork/exec [#178](https://github.com/fsnotify/fsnotify/pull/178) (thanks @pattyshack) * Linux: use InotifyInit1 with IN_CLOEXEC to stop leaking a file descriptor to a child process when using fork/exec [#178](https://github.com/fsnotify/fsnotify/pull/178) (thanks @pattyshack)
@ -79,7 +89,7 @@ kqueue: Fix logic for CREATE after REMOVE [#111](https://github.com/fsnotify/fsn
## v1.0.2 / 2014-08-17 ## v1.0.2 / 2014-08-17
* [Fix] Missing create events on OS X. [#14](https://github.com/fsnotify/fsnotify/issues/14) (thanks @zhsso) * [Fix] Missing create events on macOS. [#14](https://github.com/fsnotify/fsnotify/issues/14) (thanks @zhsso)
* [Fix] Make ./path and path equivalent. (thanks @zhsso) * [Fix] Make ./path and path equivalent. (thanks @zhsso)
## v1.0.0 / 2014-08-15 ## v1.0.0 / 2014-08-15
@ -142,7 +152,7 @@ kqueue: Fix logic for CREATE after REMOVE [#111](https://github.com/fsnotify/fsn
## v0.9.2 / 2014-08-17 ## v0.9.2 / 2014-08-17
* [Backport] Fix missing create events on OS X. [#14](https://github.com/fsnotify/fsnotify/issues/14) (thanks @zhsso) * [Backport] Fix missing create events on macOS. [#14](https://github.com/fsnotify/fsnotify/issues/14) (thanks @zhsso)
## v0.9.1 / 2014-06-12 ## v0.9.1 / 2014-06-12
@ -161,7 +171,7 @@ kqueue: Fix logic for CREATE after REMOVE [#111](https://github.com/fsnotify/fsn
## v0.8.11 / 2013-11-02 ## v0.8.11 / 2013-11-02
* [Doc] Add Changelog [#72][] (thanks @nathany) * [Doc] Add Changelog [#72][] (thanks @nathany)
* [Doc] Spotlight and double modify events on OS X [#62][] (reported by @paulhammond) * [Doc] Spotlight and double modify events on macOS [#62][] (reported by @paulhammond)
## v0.8.10 / 2013-10-19 ## v0.8.10 / 2013-10-19

View file

@ -17,7 +17,7 @@ Please indicate that you have signed the CLA in your pull request.
### How fsnotify is Developed ### How fsnotify is Developed
* Development is done on feature branches. * Development is done on feature branches.
* Tests are run on BSD, Linux, OS X and Windows. * Tests are run on BSD, Linux, macOS and Windows.
* Pull requests are reviewed and [applied to master][am] using [hub][]. * Pull requests are reviewed and [applied to master][am] using [hub][].
* Maintainers may modify or squash commits rather than asking contributors to. * Maintainers may modify or squash commits rather than asking contributors to.
* To issue a new release, the maintainers will: * To issue a new release, the maintainers will:
@ -44,7 +44,7 @@ This workflow is [thoroughly explained by Katrina Owen](https://splice.com/blog/
### Testing ### Testing
fsnotify uses build tags to compile different code on Linux, BSD, OS X, and Windows. fsnotify uses build tags to compile different code on Linux, BSD, macOS, and Windows.
Before doing a pull request, please do your best to test your changes on multiple platforms, and list which platforms you were able/unable to test on. Before doing a pull request, please do your best to test your changes on multiple platforms, and list which platforms you were able/unable to test on.
@ -58,7 +58,7 @@ To aid in cross-platform testing there is a Vagrantfile for Linux and BSD.
Notice: fsnotify file system events won't trigger in shared folders. The tests get around this limitation by using the /tmp directory. Notice: fsnotify file system events won't trigger in shared folders. The tests get around this limitation by using the /tmp directory.
Right now there is no equivalent solution for Windows and OS X, but there are Windows VMs [freely available from Microsoft](http://www.modern.ie/en-us/virtualization-tools#downloads). Right now there is no equivalent solution for Windows and macOS, but there are Windows VMs [freely available from Microsoft](http://www.modern.ie/en-us/virtualization-tools#downloads).
### Maintainers ### Maintainers

View file

@ -8,14 +8,14 @@ fsnotify utilizes [golang.org/x/sys](https://godoc.org/golang.org/x/sys) rather
go get -u golang.org/x/sys/... go get -u golang.org/x/sys/...
``` ```
Cross platform: Windows, Linux, BSD and OS X. Cross platform: Windows, Linux, BSD and macOS.
|Adapter |OS |Status | |Adapter |OS |Status |
|----------|----------|----------| |----------|----------|----------|
|inotify |Linux 2.6.27 or later, Android\*|Supported [![Build Status](https://travis-ci.org/fsnotify/fsnotify.svg?branch=master)](https://travis-ci.org/fsnotify/fsnotify)| |inotify |Linux 2.6.27 or later, Android\*|Supported [![Build Status](https://travis-ci.org/fsnotify/fsnotify.svg?branch=master)](https://travis-ci.org/fsnotify/fsnotify)|
|kqueue |BSD, OS X, iOS\*|Supported [![Build Status](https://travis-ci.org/fsnotify/fsnotify.svg?branch=master)](https://travis-ci.org/fsnotify/fsnotify)| |kqueue |BSD, macOS, iOS\*|Supported [![Build Status](https://travis-ci.org/fsnotify/fsnotify.svg?branch=master)](https://travis-ci.org/fsnotify/fsnotify)|
|ReadDirectoryChangesW|Windows|Supported [![Build status](https://ci.appveyor.com/api/projects/status/ivwjubaih4r0udeh/branch/master?svg=true)](https://ci.appveyor.com/project/NathanYoungman/fsnotify/branch/master)| |ReadDirectoryChangesW|Windows|Supported [![Build status](https://ci.appveyor.com/api/projects/status/ivwjubaih4r0udeh/branch/master?svg=true)](https://ci.appveyor.com/project/NathanYoungman/fsnotify/branch/master)|
|FSEvents |OS X |[Planned](https://github.com/fsnotify/fsnotify/issues/11)| |FSEvents |macOS |[Planned](https://github.com/fsnotify/fsnotify/issues/11)|
|FEN |Solaris 11 |[In Progress](https://github.com/fsnotify/fsnotify/issues/12)| |FEN |Solaris 11 |[In Progress](https://github.com/fsnotify/fsnotify/issues/12)|
|fanotify |Linux 2.6.37+ | | |fanotify |Linux 2.6.37+ | |
|USN Journals |Windows |[Maybe](https://github.com/fsnotify/fsnotify/issues/53)| |USN Journals |Windows |[Maybe](https://github.com/fsnotify/fsnotify/issues/53)|
@ -23,7 +23,7 @@ Cross platform: Windows, Linux, BSD and OS X.
\* Android and iOS are untested. \* Android and iOS are untested.
Please see [the documentation](https://godoc.org/github.com/fsnotify/fsnotify) for usage. Consult the [Wiki](https://github.com/fsnotify/fsnotify/wiki) for the FAQ and further information. Please see [the documentation](https://godoc.org/github.com/fsnotify/fsnotify) and consult the [FAQ](#faq) for usage information.
## API stability ## API stability
@ -41,6 +41,35 @@ Please refer to [CONTRIBUTING][] before opening an issue or pull request.
See [example_test.go](https://github.com/fsnotify/fsnotify/blob/master/example_test.go). See [example_test.go](https://github.com/fsnotify/fsnotify/blob/master/example_test.go).
## FAQ
**When a file is moved to another directory is it still being watched?**
No (it shouldn't be, unless you are watching where it was moved to).
**When I watch a directory, are all subdirectories watched as well?**
No, you must add watches for any directory you want to watch (a recursive watcher is on the roadmap [#18][]).
**Do I have to watch the Error and Event channels in a separate goroutine?**
As of now, yes. Looking into making this single-thread friendly (see [howeyc #7][#7])
**Why am I receiving multiple events for the same file on OS X?**
Spotlight indexing on OS X can result in multiple events (see [howeyc #62][#62]). A temporary workaround is to add your folder(s) to the *Spotlight Privacy settings* until we have a native FSEvents implementation (see [#11][]).
**How many files can be watched at once?**
There are OS-specific limits as to how many watches can be created:
* Linux: /proc/sys/fs/inotify/max_user_watches contains the limit, reaching this limit results in a "no space left on device" error.
* BSD / OSX: sysctl variables "kern.maxfiles" and "kern.maxfilesperproc", reaching these limits results in a "too many open files" error.
[#62]: https://github.com/howeyc/fsnotify/issues/62
[#18]: https://github.com/fsnotify/fsnotify/issues/18
[#11]: https://github.com/fsnotify/fsnotify/issues/11
[#7]: https://github.com/howeyc/fsnotify/issues/7
[contributing]: https://github.com/fsnotify/fsnotify/blob/master/CONTRIBUTING.md [contributing]: https://github.com/fsnotify/fsnotify/blob/master/CONTRIBUTING.md
## Related Projects ## Related Projects

View file

@ -9,6 +9,7 @@ package fsnotify
import ( import (
"bytes" "bytes"
"errors"
"fmt" "fmt"
) )
@ -60,3 +61,6 @@ func (op Op) String() string {
func (e Event) String() string { func (e Event) String() string {
return fmt.Sprintf("%q: %s", e.Name, e.Op.String()) return fmt.Sprintf("%q: %s", e.Name, e.Op.String())
} }
// Common errors that can be reported by a watcher
var ErrEventOverflow = errors.New("fsnotify queue overflow")

View file

@ -6,7 +6,11 @@
package fsnotify package fsnotify
import "testing" import (
"os"
"testing"
"time"
)
func TestEventStringWithValue(t *testing.T) { func TestEventStringWithValue(t *testing.T) {
for opMask, expectedString := range map[Op]string{ for opMask, expectedString := range map[Op]string{
@ -38,3 +42,29 @@ func TestEventOpStringWithNoValue(t *testing.T) {
t.Fatalf("Expected %s, got: %v", expectedOpString, event.Op.String()) t.Fatalf("Expected %s, got: %v", expectedOpString, event.Op.String())
} }
} }
// TestWatcherClose tests that the goroutine started by creating the watcher can be
// signalled to return at any time, even if there is no goroutine listening on the events
// or errors channels.
func TestWatcherClose(t *testing.T) {
t.Parallel()
name := tempMkFile(t, "")
w := newWatcher(t)
err := w.Add(name)
if err != nil {
t.Fatal(err)
}
err = os.Remove(name)
if err != nil {
t.Fatal(err)
}
// Allow the watcher to receive the event.
time.Sleep(time.Millisecond * 100)
err = w.Close()
if err != nil {
t.Fatal(err)
}
}

View file

@ -24,7 +24,6 @@ type Watcher struct {
Events chan Event Events chan Event
Errors chan error Errors chan error
mu sync.Mutex // Map access mu sync.Mutex // Map access
cv *sync.Cond // sync removing on rm_watch with IN_IGNORE
fd int fd int
poller *fdPoller poller *fdPoller
watches map[string]*watch // Map of inotify watches (key: path) watches map[string]*watch // Map of inotify watches (key: path)
@ -56,7 +55,6 @@ func NewWatcher() (*Watcher, error) {
done: make(chan struct{}), done: make(chan struct{}),
doneResp: make(chan struct{}), doneResp: make(chan struct{}),
} }
w.cv = sync.NewCond(&w.mu)
go w.readEvents() go w.readEvents()
return w, nil return w, nil
@ -103,21 +101,23 @@ func (w *Watcher) Add(name string) error {
var flags uint32 = agnosticEvents var flags uint32 = agnosticEvents
w.mu.Lock() w.mu.Lock()
watchEntry, found := w.watches[name] defer w.mu.Unlock()
w.mu.Unlock() watchEntry := w.watches[name]
if found { if watchEntry != nil {
watchEntry.flags |= flags flags |= watchEntry.flags | unix.IN_MASK_ADD
flags |= unix.IN_MASK_ADD
} }
wd, errno := unix.InotifyAddWatch(w.fd, name, flags) wd, errno := unix.InotifyAddWatch(w.fd, name, flags)
if wd == -1 { if wd == -1 {
return errno return errno
} }
w.mu.Lock() if watchEntry == nil {
w.watches[name] = &watch{wd: uint32(wd), flags: flags} w.watches[name] = &watch{wd: uint32(wd), flags: flags}
w.paths[wd] = name w.paths[wd] = name
w.mu.Unlock() } else {
watchEntry.wd = uint32(wd)
watchEntry.flags = flags
}
return nil return nil
} }
@ -135,6 +135,13 @@ func (w *Watcher) Remove(name string) error {
if !ok { if !ok {
return fmt.Errorf("can't remove non-existent inotify watch for: %s", name) return fmt.Errorf("can't remove non-existent inotify watch for: %s", name)
} }
// We successfully removed the watch if InotifyRmWatch doesn't return an
// error, we need to clean up our internal state to ensure it matches
// inotify's kernel state.
delete(w.paths, int(watch.wd))
delete(w.watches, name)
// inotify_rm_watch will return EINVAL if the file has been deleted; // inotify_rm_watch will return EINVAL if the file has been deleted;
// the inotify will already have been removed. // the inotify will already have been removed.
// watches and pathes are deleted in ignoreLinux() implicitly and asynchronously // watches and pathes are deleted in ignoreLinux() implicitly and asynchronously
@ -152,13 +159,6 @@ func (w *Watcher) Remove(name string) error {
return errno return errno
} }
// wait until ignoreLinux() deleting maps
exists := true
for exists {
w.cv.Wait()
_, exists = w.watches[name]
}
return nil return nil
} }
@ -245,13 +245,31 @@ func (w *Watcher) readEvents() {
mask := uint32(raw.Mask) mask := uint32(raw.Mask)
nameLen := uint32(raw.Len) nameLen := uint32(raw.Len)
if mask&unix.IN_Q_OVERFLOW != 0 {
select {
case w.Errors <- ErrEventOverflow:
case <-w.done:
return
}
}
// If the event happened to the watched directory or the watched file, the kernel // If the event happened to the watched directory or the watched file, the kernel
// doesn't append the filename to the event, but we would like to always fill the // doesn't append the filename to the event, but we would like to always fill the
// the "Name" field with a valid filename. We retrieve the path of the watch from // the "Name" field with a valid filename. We retrieve the path of the watch from
// the "paths" map. // the "paths" map.
w.mu.Lock() w.mu.Lock()
name := w.paths[int(raw.Wd)] name, ok := w.paths[int(raw.Wd)]
// IN_DELETE_SELF occurs when the file/directory being watched is removed.
// This is a sign to clean up the maps, otherwise we are no longer in sync
// with the inotify kernel state which has already deleted the watch
// automatically.
if ok && mask&unix.IN_DELETE_SELF == unix.IN_DELETE_SELF {
delete(w.paths, int(raw.Wd))
delete(w.watches, name)
}
w.mu.Unlock() w.mu.Unlock()
if nameLen > 0 { if nameLen > 0 {
// Point "bytes" at the first byte of the filename // Point "bytes" at the first byte of the filename
bytes := (*[unix.PathMax]byte)(unsafe.Pointer(&buf[offset+unix.SizeofInotifyEvent])) bytes := (*[unix.PathMax]byte)(unsafe.Pointer(&buf[offset+unix.SizeofInotifyEvent]))
@ -262,7 +280,7 @@ func (w *Watcher) readEvents() {
event := newEvent(name, mask) event := newEvent(name, mask)
// Send the events that are not ignored on the events channel // Send the events that are not ignored on the events channel
if !event.ignoreLinux(w, raw.Wd, mask) { if !event.ignoreLinux(mask) {
select { select {
case w.Events <- event: case w.Events <- event:
case <-w.done: case <-w.done:
@ -279,15 +297,9 @@ func (w *Watcher) readEvents() {
// Certain types of events can be "ignored" and not sent over the Events // Certain types of events can be "ignored" and not sent over the Events
// channel. Such as events marked ignore by the kernel, or MODIFY events // channel. Such as events marked ignore by the kernel, or MODIFY events
// against files that do not exist. // against files that do not exist.
func (e *Event) ignoreLinux(w *Watcher, wd int32, mask uint32) bool { func (e *Event) ignoreLinux(mask uint32) bool {
// Ignore anything the inotify API says to ignore // Ignore anything the inotify API says to ignore
if mask&unix.IN_IGNORED == unix.IN_IGNORED { if mask&unix.IN_IGNORED == unix.IN_IGNORED {
w.mu.Lock()
defer w.mu.Unlock()
name := w.paths[int(wd)]
delete(w.paths, int(wd))
delete(w.watches, name)
w.cv.Broadcast()
return true return true
} }

View file

@ -293,25 +293,23 @@ func TestInotifyRemoveTwice(t *testing.T) {
t.Fatalf("Failed to add testFile: %v", err) t.Fatalf("Failed to add testFile: %v", err)
} }
err = os.Remove(testFile) err = w.Remove(testFile)
if err != nil { if err != nil {
t.Fatalf("Failed to remove testFile: %v", err) t.Fatalf("wanted successful remove but got: %v", err)
} }
err = w.Remove(testFile) err = w.Remove(testFile)
if err == nil { if err == nil {
t.Fatalf("no error on removing invalid file") t.Fatalf("no error on removing invalid file")
} }
s1 := fmt.Sprintf("%s", err)
err = w.Remove(testFile) w.mu.Lock()
if err == nil { defer w.mu.Unlock()
t.Fatalf("no error on removing invalid file") if len(w.watches) != 0 {
t.Fatalf("Expected watches len is 0, but got: %d, %v", len(w.watches), w.watches)
} }
s2 := fmt.Sprintf("%s", err) if len(w.paths) != 0 {
t.Fatalf("Expected paths len is 0, but got: %d, %v", len(w.paths), w.paths)
if s1 != s2 {
t.Fatalf("receive different error - %s / %s", s1, s2)
} }
} }
@ -358,3 +356,94 @@ func TestInotifyInnerMapLength(t *testing.T) {
t.Fatalf("Expected paths len is 0, but got: %d, %v", len(w.paths), w.paths) t.Fatalf("Expected paths len is 0, but got: %d, %v", len(w.paths), w.paths)
} }
} }
func TestInotifyOverflow(t *testing.T) {
// We need to generate many more events than the
// fs.inotify.max_queued_events sysctl setting.
// We use multiple goroutines (one per directory)
// to speed up file creation.
numDirs := 128
numFiles := 1024
testDir := tempMkdir(t)
defer os.RemoveAll(testDir)
w, err := NewWatcher()
if err != nil {
t.Fatalf("Failed to create watcher: %v", err)
}
defer w.Close()
for dn := 0; dn < numDirs; dn++ {
testSubdir := fmt.Sprintf("%s/%d", testDir, dn)
err := os.Mkdir(testSubdir, 0777)
if err != nil {
t.Fatalf("Cannot create subdir: %v", err)
}
err = w.Add(testSubdir)
if err != nil {
t.Fatalf("Failed to add subdir: %v", err)
}
}
errChan := make(chan error, numDirs*numFiles)
for dn := 0; dn < numDirs; dn++ {
testSubdir := fmt.Sprintf("%s/%d", testDir, dn)
go func() {
for fn := 0; fn < numFiles; fn++ {
testFile := fmt.Sprintf("%s/%d", testSubdir, fn)
handle, err := os.Create(testFile)
if err != nil {
errChan <- fmt.Errorf("Create failed: %v", err)
continue
}
err = handle.Close()
if err != nil {
errChan <- fmt.Errorf("Close failed: %v", err)
continue
}
}
}()
}
creates := 0
overflows := 0
after := time.After(10 * time.Second)
for overflows == 0 && creates < numDirs*numFiles {
select {
case <-after:
t.Fatalf("Not done")
case err := <-errChan:
t.Fatalf("Got an error from file creator goroutine: %v", err)
case err := <-w.Errors:
if err == ErrEventOverflow {
overflows++
} else {
t.Fatalf("Got an error from watcher: %v", err)
}
case evt := <-w.Events:
if !strings.HasPrefix(evt.Name, testDir) {
t.Fatalf("Got an event for an unknown file: %s", evt.Name)
}
if evt.Op == Create {
creates++
}
}
}
if creates == numDirs*numFiles {
t.Fatalf("Could not trigger overflow")
}
if overflows == 0 {
t.Fatalf("No overflow and not enough creates (expected %d, got %d)",
numDirs*numFiles, creates)
}
}

View file

@ -13,9 +13,9 @@ import (
"golang.org/x/sys/unix" "golang.org/x/sys/unix"
) )
// testExchangedataForWatcher tests the watcher with the exchangedata operation on OS X. // testExchangedataForWatcher tests the watcher with the exchangedata operation on macOS.
// //
// This is widely used for atomic saves on OS X, e.g. TextMate and in Apple's NSDocument. // This is widely used for atomic saves on macOS, e.g. TextMate and in Apple's NSDocument.
// //
// See https://developer.apple.com/library/mac/documentation/Darwin/Reference/ManPages/man2/exchangedata.2.html // See https://developer.apple.com/library/mac/documentation/Darwin/Reference/ManPages/man2/exchangedata.2.html
// Also see: https://github.com/textmate/textmate/blob/cd016be29489eba5f3c09b7b70b06da134dda550/Frameworks/io/src/swap_file_data.cc#L20 // Also see: https://github.com/textmate/textmate/blob/cd016be29489eba5f3c09b7b70b06da134dda550/Frameworks/io/src/swap_file_data.cc#L20

View file

@ -22,7 +22,7 @@ import (
type Watcher struct { type Watcher struct {
Events chan Event Events chan Event
Errors chan error Errors chan error
done chan bool // Channel for sending a "quit message" to the reader goroutine done chan struct{} // Channel for sending a "quit message" to the reader goroutine
kq int // File descriptor (as returned by the kqueue() syscall). kq int // File descriptor (as returned by the kqueue() syscall).
@ -56,7 +56,7 @@ func NewWatcher() (*Watcher, error) {
externalWatches: make(map[string]bool), externalWatches: make(map[string]bool),
Events: make(chan Event), Events: make(chan Event),
Errors: make(chan error), Errors: make(chan error),
done: make(chan bool), done: make(chan struct{}),
} }
go w.readEvents() go w.readEvents()
@ -71,10 +71,8 @@ func (w *Watcher) Close() error {
return nil return nil
} }
w.isClosed = true w.isClosed = true
w.mu.Unlock()
// copy paths to remove while locked // copy paths to remove while locked
w.mu.Lock()
var pathsToRemove = make([]string, 0, len(w.watches)) var pathsToRemove = make([]string, 0, len(w.watches))
for name := range w.watches { for name := range w.watches {
pathsToRemove = append(pathsToRemove, name) pathsToRemove = append(pathsToRemove, name)
@ -82,15 +80,12 @@ func (w *Watcher) Close() error {
w.mu.Unlock() w.mu.Unlock()
// unlock before calling Remove, which also locks // unlock before calling Remove, which also locks
var err error
for _, name := range pathsToRemove { for _, name := range pathsToRemove {
if e := w.Remove(name); e != nil && err == nil { w.Remove(name)
err = e
}
} }
// Send "quit" message to the reader goroutine: // send a "quit" message to the reader goroutine
w.done <- true close(w.done)
return nil return nil
} }
@ -266,17 +261,12 @@ func (w *Watcher) addWatch(name string, flags uint32) (string, error) {
func (w *Watcher) readEvents() { func (w *Watcher) readEvents() {
eventBuffer := make([]unix.Kevent_t, 10) eventBuffer := make([]unix.Kevent_t, 10)
loop:
for { for {
// See if there is a message on the "done" channel // See if there is a message on the "done" channel
select { select {
case <-w.done: case <-w.done:
err := unix.Close(w.kq) break loop
if err != nil {
w.Errors <- err
}
close(w.Events)
close(w.Errors)
return
default: default:
} }
@ -284,7 +274,11 @@ func (w *Watcher) readEvents() {
kevents, err := read(w.kq, eventBuffer, &keventWaitTime) kevents, err := read(w.kq, eventBuffer, &keventWaitTime)
// EINTR is okay, the syscall was interrupted before timeout expired. // EINTR is okay, the syscall was interrupted before timeout expired.
if err != nil && err != unix.EINTR { if err != nil && err != unix.EINTR {
w.Errors <- err select {
case w.Errors <- err:
case <-w.done:
break loop
}
continue continue
} }
@ -319,8 +313,12 @@ func (w *Watcher) readEvents() {
if path.isDir && event.Op&Write == Write && !(event.Op&Remove == Remove) { if path.isDir && event.Op&Write == Write && !(event.Op&Remove == Remove) {
w.sendDirectoryChangeEvents(event.Name) w.sendDirectoryChangeEvents(event.Name)
} else { } else {
// Send the event on the Events channel // Send the event on the Events channel.
w.Events <- event select {
case w.Events <- event:
case <-w.done:
break loop
}
} }
if event.Op&Remove == Remove { if event.Op&Remove == Remove {
@ -352,6 +350,18 @@ func (w *Watcher) readEvents() {
kevents = kevents[1:] kevents = kevents[1:]
} }
} }
// cleanup
err := unix.Close(w.kq)
if err != nil {
// only way the previous loop breaks is if w.done was closed so we need to async send to w.Errors.
select {
case w.Errors <- err:
default:
}
}
close(w.Events)
close(w.Errors)
} }
// newEvent returns an platform-independent Event based on kqueue Fflags. // newEvent returns an platform-independent Event based on kqueue Fflags.
@ -407,7 +417,11 @@ func (w *Watcher) sendDirectoryChangeEvents(dirPath string) {
// Get all files // Get all files
files, err := ioutil.ReadDir(dirPath) files, err := ioutil.ReadDir(dirPath)
if err != nil { if err != nil {
w.Errors <- err select {
case w.Errors <- err:
case <-w.done:
return
}
} }
// Search for new files // Search for new files
@ -428,7 +442,11 @@ func (w *Watcher) sendFileCreatedEventIfNew(filePath string, fileInfo os.FileInf
w.mu.Unlock() w.mu.Unlock()
if !doesExist { if !doesExist {
// Send create event // Send create event
w.Events <- newCreateEvent(filePath) select {
case w.Events <- newCreateEvent(filePath):
case <-w.done:
return
}
} }
// like watchDirectoryFiles (but without doing another ReadDir) // like watchDirectoryFiles (but without doing another ReadDir)