1 #ifndef _SPARC64_SEMAPHORE_H
2 #define _SPARC64_SEMAPHORE_H
3
4 /* These are actually reasonable on the V9. */
5 #ifdef __KERNEL__
6
7 #include <asm/atomic.h>
8 #include <asm/bitops.h>
9 #include <asm/system.h>
10 #include <linux/wait.h>
11
12 struct semaphore {
13 atomic_t count;
14 atomic_t waking;
15 wait_queue_head_t wait;
16 #if WAITQUEUE_DEBUG
17 long __magic;
18 #endif
19 };
20
21 #if WAITQUEUE_DEBUG
22 # define __SEM_DEBUG_INIT(name) \
23 , (long)&(name).__magic
24 #else
25 # define __SEM_DEBUG_INIT(name)
26 #endif
27
28 #define __SEMAPHORE_INITIALIZER(name,count) \
29 { ATOMIC_INIT(count), ATOMIC_INIT(0), __WAIT_QUEUE_HEAD_INITIALIZER((name).wait) \
30 __SEM_DEBUG_INIT(name) }
31
32 #define __MUTEX_INITIALIZER(name) \
33 __SEMAPHORE_INITIALIZER(name,1)
34
35 #define __DECLARE_SEMAPHORE_GENERIC(name,count) \
36 struct semaphore name = __SEMAPHORE_INITIALIZER(name,count)
37
38 #define DECLARE_MUTEX(name) __DECLARE_SEMAPHORE_GENERIC(name,1)
39 #define DECLARE_MUTEX_LOCKED(name) __DECLARE_SEMAPHORE_GENERIC(name,0)
40
41 extern inline void sema_init (struct semaphore *sem, int val)
42 {
43 atomic_set(&sem->count, val);
44 atomic_set(&sem->waking, 0);
45 init_waitqueue_head(&sem->wait);
46 #if WAITQUEUE_DEBUG
47 sem->__magic = (long)&sem->__magic;
48 #endif
49 }
50
51 static inline void init_MUTEX (struct semaphore *sem)
52 {
53 sema_init(sem, 1);
54 }
55
56 static inline void init_MUTEX_LOCKED (struct semaphore *sem)
57 {
58 sema_init(sem, 0);
59 }
60
61 extern void __down(struct semaphore * sem);
62 extern int __down_interruptible(struct semaphore * sem);
63 extern int __down_trylock(struct semaphore * sem);
64 extern void __up(struct semaphore * sem);
65
66 extern __inline__ void down(struct semaphore * sem)
67 {
68 #if WAITQUEUE_DEBUG
69 CHECK_MAGIC(sem->__magic);
70 #endif
71 __asm__ __volatile__("
72 1: lduw [%0], %%g5
73 sub %%g5, 1, %%g7
74 cas [%0], %%g5, %%g7
75 cmp %%g5, %%g7
76 bne,pn %%icc, 1b
77 cmp %%g7, 1
78 bl,pn %%icc, 3f
79 membar #StoreStore
80 2:
81 .subsection 2
82 3: mov %0, %%g5
83 save %%sp, -160, %%sp
84 mov %%g1, %%l1
85 mov %%g2, %%l2
86 mov %%g3, %%l3
87 call %1
88 mov %%g5, %%o0
89 mov %%l1, %%g1
90 mov %%l2, %%g2
91 ba,pt %%xcc, 2b
92 restore %%l3, %%g0, %%g3
93 .previous\n"
94 : : "r" (sem), "i" (__down)
95 : "g5", "g7", "memory", "cc");
96 }
97
98 extern __inline__ int down_interruptible(struct semaphore *sem)
99 {
100 int ret = 0;
101
102 #if WAITQUEUE_DEBUG
103 CHECK_MAGIC(sem->__magic);
104 #endif
105 __asm__ __volatile__("
106 1: lduw [%2], %%g5
107 sub %%g5, 1, %%g7
108 cas [%2], %%g5, %%g7
109 cmp %%g5, %%g7
110 bne,pn %%icc, 1b
111 cmp %%g7, 1
112 bl,pn %%icc, 3f
113 membar #StoreStore
114 2:
115 .subsection 2
116 3: mov %2, %%g5
117 save %%sp, -160, %%sp
118 mov %%g1, %%l1
119 mov %%g2, %%l2
120 mov %%g3, %%l3
121 call %3
122 mov %%g5, %%o0
123 mov %%l1, %%g1
124 mov %%l2, %%g2
125 mov %%l3, %%g3
126 ba,pt %%xcc, 2b
127 restore %%o0, %%g0, %0
128 .previous\n"
129 : "=r" (ret)
130 : "" (ret), "r" (sem), "i" (__down_interruptible)
131 : "g5", "g7", "memory", "cc");
132 return ret;
133 }
134
135 extern inline int down_trylock(struct semaphore *sem)
136 {
137 int ret = 0;
138 #if WAITQUEUE_DEBUG
139 CHECK_MAGIC(sem->__magic);
140 #endif
141 __asm__ __volatile__("
142 1: lduw [%2], %%g5
143 sub %%g5, 1, %%g7
144 cas [%2], %%g5, %%g7
145 cmp %%g5, %%g7
146 bne,pn %%icc, 1b
147 cmp %%g7, 1
148 bl,pn %%icc, 3f
149 membar #StoreStore
150 2:
151 .subsection 2
152 3: mov %2, %%g5
153 save %%sp, -160, %%sp
154 mov %%g1, %%l1
155 mov %%g2, %%l2
156 mov %%g3, %%l3
157 call %3
158 mov %%g5, %%o0
159 mov %%l1, %%g1
160 mov %%l2, %%g2
161 mov %%l3, %%g3
162 ba,pt %%xcc, 2b
163 restore %%o0, %%g0, %0
164 .previous\n"
165 : "=r" (ret)
166 : "" (ret), "r" (sem), "i" (__down_trylock)
167 : "g5", "g7", "memory", "cc");
168 return ret;
169 }
170
171 extern __inline__ void up(struct semaphore * sem)
172 {
173 #if WAITQUEUE_DEBUG
174 CHECK_MAGIC(sem->__magic);
175 #endif
176 __asm__ __volatile__("
177 membar #StoreLoad | #LoadLoad
178 1: lduw [%0], %%g5
179 add %%g5, 1, %%g7
180 cas [%0], %%g5, %%g7
181 cmp %%g5, %%g7
182 bne,pn %%icc, 1b
183 addcc %%g7, 1, %%g0
184 ble,pn %%icc, 3f
185 nop
186 2:
187 .subsection 2
188 3: mov %0, %%g5
189 save %%sp, -160, %%sp
190 mov %%g1, %%l1
191 mov %%g2, %%l2
192 mov %%g3, %%l3
193 call %1
194 mov %%g5, %%o0
195 mov %%l1, %%g1
196 mov %%l2, %%g2
197 ba,pt %%xcc, 2b
198 restore %%l3, %%g0, %%g3
199 .previous\n"
200 : : "r" (sem), "i" (__up)
201 : "g5", "g7", "memory", "cc");
202 }
203
204 /* rw mutexes (should that be mutices? =) -- throw rw
205 * spinlocks and semaphores together, and this is what we
206 * end up with...
207 *
208 * The lock is initialized to BIAS. This way, a writer
209 * subtracts BIAS ands gets 0 for the case of an uncontended
210 * lock. Readers decrement by 1 and see a positive value
211 * when uncontended, negative if there are writers waiting
212 * (in which case it goes to sleep).
213 *
214 * The value 0x01000000 supports up to 128 processors and
215 * lots of processes. BIAS must be chosen such that subtracting
216 * BIAS once per CPU will result in the int remaining
217 * negative.
218 * In terms of fairness, this should result in the lock
219 * flopping back and forth between readers and writers
220 * under heavy use.
221 *
222 * -ben
223 *
224 * Once we start supporting machines with more than 128 CPUs,
225 * we should go for using a 64bit atomic type instead of 32bit
226 * as counter. We shall probably go for bias 0x80000000 then,
227 * so that single sethi can set it.
228 *
229 * -jj
230 */
231 #define RW_LOCK_BIAS 0x01000000
232 #define RW_LOCK_BIAS_STR "0x01000000"
233
234 struct rw_semaphore {
235 int count;
236 /* So that this does not have to be 64bit type,
237 * we'll use le bitops on it which use casa instead of casx.
238 * bit 0 means read bias granted
239 * bit 1 means write bias granted
240 */
241 unsigned granted;
242 wait_queue_head_t wait;
243 wait_queue_head_t write_bias_wait;
244 #if WAITQUEUE_DEBUG
245 long __magic;
246 atomic_t readers;
247 atomic_t writers;
248 #endif
249 };
250
251 #if WAITQUEUE_DEBUG
252 #define __RWSEM_DEBUG_INIT , ATOMIC_INIT(0), ATOMIC_INIT(0)
253 #else
254 #define __RWSEM_DEBUG_INIT /* */
255 #endif
256
257 #define __RWSEM_INITIALIZER(name,count) \
258 { (count), 0, __WAIT_QUEUE_HEAD_INITIALIZER((name).wait), \
259 __WAIT_QUEUE_HEAD_INITIALIZER((name).write_bias_wait) \
260 __SEM_DEBUG_INIT(name) __RWSEM_DEBUG_INIT }
261
262 #define __DECLARE_RWSEM_GENERIC(name,count) \
263 struct rw_semaphore name = __RWSEM_INITIALIZER(name,count)
264
265 #define DECLARE_RWSEM(name) __DECLARE_RWSEM_GENERIC(name,RW_LOCK_BIAS)
266 #define DECLARE_RWSEM_READ_LOCKED(name) __DECLARE_RWSEM_GENERIC(name,RW_LOCK_BIAS-1)
267 #define DECLARE_RWSEM_WRITE_LOCKED(name) __DECLARE_RWSEM_GENERIC(name,0)
268
269 extern inline void init_rwsem(struct rw_semaphore *sem)
270 {
271 sem->count = RW_LOCK_BIAS;
272 sem->granted = 0;
273 init_waitqueue_head(&sem->wait);
274 init_waitqueue_head(&sem->write_bias_wait);
275 #if WAITQUEUE_DEBUG
276 sem->__magic = (long)&sem->__magic;
277 atomic_set(&sem->readers, 0);
278 atomic_set(&sem->writers, 0);
279 #endif
280 }
281
282 extern void __down_read_failed(/* Special calling convention */ void);
283 extern void __down_write_failed(/* Special calling convention */ void);
284 extern void __rwsem_wake(struct rw_semaphore *sem, unsigned long readers);
285
286 extern inline void down_read(struct rw_semaphore *sem)
287 {
288 #if WAITQUEUE_DEBUG
289 CHECK_MAGIC(sem->__magic);
290 #endif
291 __asm__ __volatile__("
292 1: lduw [%0], %%g5
293 subcc %%g5, 1, %%g7
294 cas [%0], %%g5, %%g7
295 bneg,pn %%icc, 3f
296 cmp %%g5, %%g7
297 bne,pn %%icc, 1b
298 membar #StoreStore
299 2:
300 .subsection 2
301 3: bne,pn %%icc, 1b
302 mov %0, %%g7
303 save %%sp, -160, %%sp
304 mov %%g1, %%l1
305 mov %%g2, %%l2
306 call %1
307 mov %%g3, %%l3
308 mov %%l1, %%g1
309 mov %%l2, %%g2
310 ba,pt %%xcc, 2b
311 restore %%l3, %%g0, %%g3
312 .previous\n"
313 : : "r" (sem), "i" (__down_read_failed)
314 : "g5", "g7", "memory", "cc");
315 #if WAITQUEUE_DEBUG
316 if (test_le_bit(1, &sem->granted))
317 BUG();
318 if (atomic_read(&sem->writers))
319 BUG();
320 atomic_inc(&sem->readers);
321 #endif
322 }
323
324 extern inline void down_write(struct rw_semaphore *sem)
325 {
326 #if WAITQUEUE_DEBUG
327 CHECK_MAGIC(sem->__magic);
328 #endif
329 __asm__ __volatile__("
330 1: lduw [%0], %%g5
331 sethi %%hi(" RW_LOCK_BIAS_STR "), %%g7
332 subcc %%g5, %%g7, %%g7
333 cas [%0], %%g5, %%g7
334 bne,pn %%icc, 3f
335 cmp %%g5, %%g7
336 bne,pn %%icc, 1b
337 membar #StoreStore
338 2:
339 .subsection 2
340 3: bne,pn %%icc, 1b
341 mov %0, %%g7
342 save %%sp, -160, %%sp
343 mov %%g1, %%l1
344 mov %%g2, %%l2
345 call %1
346 mov %%g3, %%l3
347 mov %%l1, %%g1
348 mov %%l2, %%g2
349 ba,pt %%xcc, 2b
350 restore %%l3, %%g0, %%g3
351 .previous\n"
352 : : "r" (sem), "i" (__down_write_failed)
353 : "g5", "g7", "memory", "cc");
354 #if WAITQUEUE_DEBUG
355 if (atomic_read(&sem->writers))
356 BUG();
357 if (atomic_read(&sem->readers))
358 BUG();
359 if (test_le_bit(0, &sem->granted))
360 BUG();
361 if (test_le_bit(1, &sem->granted))
362 BUG();
363 atomic_inc(&sem->writers);
364 #endif
365 }
366
367 /* When a reader does a release, the only significant
368 * case is when there was a writer waiting, and we've
369 * bumped the count to 0: we must wake the writer up.
370 */
371 extern inline void __up_read(struct rw_semaphore *sem)
372 {
373 __asm__ __volatile__("
374 membar #StoreLoad | #LoadLoad
375 1: lduw [%0], %%g5
376 addcc %%g5, 1, %%g7
377 cas [%0], %%g5, %%g7
378 be,pn %%icc, 3f
379 cmp %%g5, %%g7
380 bne,pn %%icc, 1b
381 nop
382 2:
383 .subsection 2
384 3: bne,pn %%icc, 1b
385 mov %0, %%g7
386 save %%sp, -160, %%sp
387 mov %%g1, %%l1
388 mov %%g2, %%l2
389 clr %%o1
390 mov %%g7, %%o0
391 call %1
392 mov %%g3, %%l3
393 mov %%l1, %%g1
394 mov %%l2, %%g2
395 ba,pt %%xcc, 2b
396 restore %%l3, %%g0, %%g3
397 .previous\n"
398 : : "r" (sem), "i" (__rwsem_wake)
399 : "g5", "g7", "memory", "cc");
400 }
401
402 /* releasing the writer is easy -- just release it and
403 * wake up any sleepers.
404 */
405 extern inline void __up_write(struct rw_semaphore *sem)
406 {
407 __asm__ __volatile__("
408 membar #StoreLoad | #LoadLoad
409 1: lduw [%0], %%g5
410 sethi %%hi(" RW_LOCK_BIAS_STR "), %%g7
411 add %%g5, %%g7, %%g7
412 cas [%0], %%g5, %%g7
413 cmp %%g5, %%g7
414 bne,pn %%icc, 1b
415 sethi %%hi(" RW_LOCK_BIAS_STR "), %%g7
416 addcc %%g5, %%g7, %%g5
417 bcs,pn %%icc, 3f
418 nop
419 2:
420 .subsection 2
421 3: mov %0, %%g7
422 save %%sp, -160, %%sp
423 mov %%g1, %%l1
424 mov %%g2, %%l2
425 srl %%g5, 0, %%o1
426 mov %%g7, %%o0
427 call %1
428 mov %%g3, %%l3
429 mov %%l1, %%g1
430 mov %%l2, %%g2
431 ba,pt %%xcc, 2b
432 restore %%l3, %%g0, %%g3
433 .previous\n"
434 : : "r" (sem), "i" (__rwsem_wake)
435 : "g5", "g7", "memory", "cc");
436 }
437
438 extern inline void up_read(struct rw_semaphore *sem)
439 {
440 #if WAITQUEUE_DEBUG
441 if (test_le_bit(1, &sem->granted))
442 BUG();
443 if (atomic_read(&sem->writers))
444 BUG();
445 atomic_dec(&sem->readers);
446 #endif
447 __up_read(sem);
448 }
449
450 extern inline void up_write(struct rw_semaphore *sem)
451 {
452 #if WAITQUEUE_DEBUG
453 if (test_le_bit(0, &sem->granted))
454 BUG();
455 if (test_le_bit(1, &sem->granted))
456 BUG();
457 if (atomic_read(&sem->readers))
458 BUG();
459 if (atomic_read(&sem->writers) != 1)
460 BUG();
461 atomic_dec(&sem->writers);
462 #endif
463 __up_write(sem);
464 }
465
466 #endif /* __KERNEL__ */
467
468 #endif /* !(_SPARC64_SEMAPHORE_H) */
469
This page was automatically generated by the
LXR engine.
Visit the LXR main site for more
information.